Add support for old forks. (#191)

* Add all forks to revm
* EXTCODESIZE and EXTCODEHASH old hardfork gas fix
* EXTCODECOPY fix gas hardfork
* EIP-161 is in SPURIOUS_DRAGON hardfork
* EIP-161 create nonce increment for SPURIOUS_DRAGON
* Enable SPURIOUS_DRAGON tests
* Change database traits to return Result<Option<>>
* db result compiled and new forks passing
* not_existing, precompile perf is_cold
* passing most of legact tests
* Remove spurious precompile hotfix for old forks
* EIP-2 OOG if crate bytecode can't be paid
* Add legacy tests to github ci, fmt,clippy
* Propagate FatalExternalError
* Add Error associated type to Database.
* Small cleanup

.github/workflows/ethereum-tests.yml

@@ -21,6 +21,7 @@ jobs:
         with:
           repository: ethereum/tests
           path: ethtests
+          submodules: recursive,
 
       - name: Install toolchain
         uses: actions-rs/toolchain@v1
@@ -34,4 +35,4 @@ jobs:
           cache-on-failure: true
 
       - name: Run Ethereum tests
-        run: cargo run --profile ethtests -p revme -- statetest ethtests/GeneralStateTests
+        run: cargo run --profile ethtests -p revme -- statetest ethtests/GeneralStateTests ethtests/LegacyTests/Constantinople/GeneralStateTests/

README.md

@@ -45,6 +45,6 @@ run tests with command: `cargo run --release -- statetest tests/GeneralStateTest
 
 There is public telegram group: https://t.me/+Ig4WDWOzikA3MzA0
 
-Or you can contact me directly on email: dragan0rakita@gmail.com
+Or if you want to hire me or contact me directly, here is my email: dragan0rakita@gmail.com and telegram: https://t.me/draganrakita
 
 

bins/revme/src/statetest/cmd.rs

@@ -6,13 +6,16 @@ use structopt::StructOpt;
 #[derive(StructOpt, Debug)]
 pub struct Cmd {
     #[structopt(required = true)]
-    path: PathBuf,
+    path: Vec<PathBuf>,
 }
 
 impl Cmd {
     pub fn run(&self) -> Result<(), TestError> {
-        let test_files = find_all_json_tests(&self.path);
-        println!("Start running tests on: {:?}", self.path);
-        run(test_files)
+        for path in &self.path {
+            println!("Start running tests on: {:?}", path);
+            let test_files = find_all_json_tests(path);
+            run(test_files)?
+        }
+        Ok(())
     }
 }

bins/revme/src/statetest/models/deserializer.rs

@@ -75,3 +75,14 @@ where
         .map_err(D::Error::custom)?
         .into())
 }
+
+pub fn deserialize_opt_str_as_bytes<'de, D>(deserializer: D) -> Result<Option<Bytes>, D::Error>
+where
+    D: de::Deserializer<'de>,
+{
+    #[derive(Debug, Deserialize)]
+    struct WrappedValue(#[serde(deserialize_with = "deserialize_str_as_bytes")] Bytes);
+
+    Option::<WrappedValue>::deserialize(deserializer)
+        .map(|opt_wrapped: Option<WrappedValue>| opt_wrapped.map(|wrapped: WrappedValue| wrapped.0))
+}

bins/revme/src/statetest/models/mod.rs

@@ -30,8 +30,9 @@ pub struct Test {
     pub indexes: TxPartIndices,
     // logs
     pub logs: H256,
-    #[serde(deserialize_with = "deserialize_str_as_bytes")]
-    pub txbytes: Bytes,
+    #[serde(default)]
+    #[serde(deserialize_with = "deserialize_opt_str_as_bytes")]
+    pub txbytes: Option<Bytes>,
 }
 
 #[derive(Debug, PartialEq, Eq, Deserialize)]

bins/revme/src/statetest/models/spec.rs

@@ -3,34 +3,44 @@ use serde_derive::*;
 
 #[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Deserialize)]
 pub enum SpecName {
-    EIP150,
-    EIP158,
     Frontier,
-    Homestead,
-    Byzantium,
-    Constantinople,
-    ConstantinopleFix,
-    Istanbul,
-    EIP158ToByzantiumAt5,
     FrontierToHomesteadAt5,
+    Homestead,
     HomesteadToDaoAt5,
     HomesteadToEIP150At5,
-    ByzantiumToConstantinopleAt5,
+    EIP150,
+    EIP158, // EIP-161: State trie clearing
+    EIP158ToByzantiumAt5,
+    Byzantium,                    // done
+    ByzantiumToConstantinopleAt5, // SKIPPED
     ByzantiumToConstantinopleFixAt5,
-    Berlin,
-    London,
-    BerlinToLondonAt5,
-    Merge,
+    Constantinople, // SKIPPED
+    ConstantinopleFix,
+    Istanbul,
+    Berlin,            //done
+    BerlinToLondonAt5, // done
+    London,            // done
+    Merge,             //done
 }
 
 impl SpecName {
     pub fn to_spec_id(&self) -> SpecId {
         match self {
-            Self::Merge => SpecId::MERGE,
-            Self::London => SpecId::LONDON,
-            Self::Berlin => SpecId::BERLIN,
+            Self::Frontier => SpecId::FRONTIER,
+            Self::Homestead | Self::FrontierToHomesteadAt5 => SpecId::HOMESTEAD,
+            Self::EIP150 | Self::HomesteadToDaoAt5 | Self::HomesteadToEIP150At5 => {
+                SpecId::TANGERINE
+            }
+            Self::EIP158 => SpecId::SPURIOUS_DRAGON,
+            Self::Byzantium | Self::EIP158ToByzantiumAt5 => SpecId::BYZANTIUM,
+            Self::ConstantinopleFix | Self::ByzantiumToConstantinopleFixAt5 => SpecId::PETERSBURG,
             Self::Istanbul => SpecId::ISTANBUL,
-            _ => panic!("Conversion failed"),
+            Self::Berlin => SpecId::BERLIN,
+            Self::London | Self::BerlinToLondonAt5 => SpecId::LONDON,
+            Self::Merge => SpecId::MERGE,
+            Self::ByzantiumToConstantinopleAt5 | Self::Constantinople => {
+                panic!("Overriden with PETERSBURG")
+            } //_ => panic!("Conversion failed"),
         }
     }
 }

bins/revme/src/statetest/runner.rs

@@ -35,6 +35,8 @@ pub enum TestError {
     SerdeDeserialize(#[from] serde_json::Error),
     #[error("Internal system error")]
     SystemError,
+    #[error("Unknown private key: {private_key:?}")]
+    UnknownPrivateKey { private_key: H256 },
 }
 
 pub fn find_all_json_tests(path: &Path) -> Vec<PathBuf> {
@@ -112,6 +114,11 @@ pub fn execute_test_suit(path: &Path, elapsed: &Arc<Mutex<Duration>>) -> Result<
                 .unwrap(),
             H160::from_str("0x3fb1cd2cd96c6d5c0b5eb3322d807b34482481d4").unwrap(),
         ),
+        (
+            H256::from_str("0xfe13266ff57000135fb9aa854bbfe455d8da85b21f626307bf3263a0c2a8e7fe")
+                .unwrap(),
+            H160::from_str("0xdcc5ba93a1ed7e045690d722f2bf460a51c61415").unwrap(),
+        ),
     ]
     .into_iter()
     .collect();
@@ -129,7 +136,7 @@ pub fn execute_test_suit(path: &Path, elapsed: &Arc<Mutex<Duration>>) -> Result<
             database.insert_account_info(*address, acc_info);
             // insert storage:
             for (&slot, &value) in info.storage.iter() {
-                database.insert_account_storage(*address, slot, value)
+                let _ = database.insert_account_storage(*address, slot, value);
             }
         }
         let mut env = Env::default();
@@ -145,9 +152,13 @@ pub fn execute_test_suit(path: &Path, elapsed: &Arc<Mutex<Duration>>) -> Result<
         env.block.difficulty = unit.env.current_difficulty;
 
         //tx env
-        env.tx.caller = *map_caller_keys
-            .get(&unit.transaction.secret_key.unwrap())
-            .unwrap();
+        env.tx.caller =
+            if let Some(caller) = map_caller_keys.get(&unit.transaction.secret_key.unwrap()) {
+                *caller
+            } else {
+                let private_key = unit.transaction.secret_key.unwrap();
+                return Err(TestError::UnknownPrivateKey { private_key });
+            };
         env.tx.gas_price = unit
             .transaction
             .gas_price
@@ -156,9 +167,9 @@ pub fn execute_test_suit(path: &Path, elapsed: &Arc<Mutex<Duration>>) -> Result<
 
         // post and execution
         for (spec_name, tests) in unit.post {
-            if !matches!(
+            if matches!(
                 spec_name,
-                SpecName::Merge | SpecName::London | SpecName::Berlin | SpecName::Istanbul
+                SpecName::ByzantiumToConstantinopleAt5 | SpecName::Constantinople
             ) {
                 continue;
             }
@@ -226,13 +237,16 @@ pub fn execute_test_suit(path: &Path, elapsed: &Arc<Mutex<Duration>>) -> Result<
 
                 *elapsed.lock().unwrap() += timer;
 
+                let is_legacy = !SpecId::enabled(evm.env.cfg.spec_id, SpecId::SPURIOUS_DRAGON);
                 let db = evm.db().unwrap();
                 let state_root = state_merkle_trie_root(
                     db.accounts
                         .iter()
                         .filter(|(_address, acc)| {
-                            !(acc.info.is_empty())
-                                || matches!(acc.account_state, AccountState::None)
+                            (is_legacy && !matches!(acc.account_state, AccountState::NotExisting))
+                                || (!is_legacy
+                                    && (!(acc.info.is_empty())
+                                        || matches!(acc.account_state, AccountState::None)))
                         })
                         .map(|(k, v)| (*k, v.clone())),
                 );
@@ -269,7 +283,7 @@ pub fn execute_test_suit(path: &Path, elapsed: &Arc<Mutex<Duration>>) -> Result<
 pub fn run(test_files: Vec<PathBuf>) -> Result<(), TestError> {
     let endjob = Arc::new(AtomicBool::new(false));
     let console_bar = Arc::new(ProgressBar::new(test_files.len() as u64));
-    let mut joins = Vec::new();
+    let mut joins: Vec<std::thread::JoinHandle<Result<(), TestError>>> = Vec::new();
     let queue = Arc::new(Mutex::new((0, test_files)));
     let elapsed = Arc::new(Mutex::new(std::time::Duration::ZERO));
     for _ in 0..10 {

bins/revme/src/statetest/trace.rs

@@ -58,7 +58,7 @@ impl<DB: Database> Inspector<DB> for CustomPrintTracer {
         let gas_remaining = interp.gas.remaining() + self.full_gas_block - self.reduced_gas_block;
 
         println!(
-            "depth:{}, PC:{}, gas:{:#x}({}), OPCODE: {:?}({:?})  refund:{:#x}({}) Stack:{:?}, Data:",
+            "depth:{}, PC:{}, gas:{:#x}({}), OPCODE: {:?}({:?})  refund:{:#x}({}) Stack:{:?}, Data size:{}",
             data.journaled_state.depth(),
             interp.program_counter(),
             gas_remaining,
@@ -68,7 +68,7 @@ impl<DB: Database> Inspector<DB> for CustomPrintTracer {
             interp.gas.refunded(),
             interp.gas.refunded(),
             interp.stack.data(),
-            //hex::encode(interp.memory.data()),
+            interp.memory.data().len(),
         );
 
         let pc = interp.program_counter();
@@ -141,12 +141,12 @@ impl<DB: Database> Inspector<DB> for CustomPrintTracer {
         is_static: bool,
     ) -> (Return, Gas, Bytes) {
         println!(
-            "SM CALL:   {:?},context:{:?}, is_static:{:?}, transfer:{:?}, input:{:?}",
+            "SM CALL:   {:?},context:{:?}, is_static:{:?}, transfer:{:?}, input_size:{:?}",
             inputs.contract,
             inputs.context,
             is_static,
             inputs.transfer,
-            hex::encode(&inputs.input),
+            inputs.input.len(),
         );
         (Return::Continue, Gas::new(0), Bytes::new())
     }

crates/revm/src/db.rs

@@ -16,15 +16,16 @@ use auto_impl::auto_impl;
 
 #[auto_impl(& mut, Box)]
 pub trait Database {
+    type Error;
     /// Get basic account information.
-    fn basic(&mut self, address: H160) -> AccountInfo;
+    fn basic(&mut self, address: H160) -> Result<Option<AccountInfo>, Self::Error>;
     /// Get account code by its hash
-    fn code_by_hash(&mut self, code_hash: H256) -> Bytecode;
+    fn code_by_hash(&mut self, code_hash: H256) -> Result<Bytecode, Self::Error>;
     /// Get storage value of address at index.
-    fn storage(&mut self, address: H160, index: U256) -> U256;
+    fn storage(&mut self, address: H160, index: U256) -> Result<U256, Self::Error>;
 
     // History related
-    fn block_hash(&mut self, number: U256) -> H256;
+    fn block_hash(&mut self, number: U256) -> Result<H256, Self::Error>;
 }
 
 #[auto_impl(& mut, Box)]
@@ -34,49 +35,47 @@ pub trait DatabaseCommit {
 
 #[auto_impl(&, Box)]
 pub trait DatabaseRef {
+    type Error;
     /// Whether account at address exists.
     //fn exists(&self, address: H160) -> Option<AccountInfo>;
     /// Get basic account information.
-    fn basic(&self, address: H160) -> AccountInfo;
+    fn basic(&self, address: H160) -> Result<Option<AccountInfo>, Self::Error>;
     /// Get account code by its hash
-    fn code_by_hash(&self, code_hash: H256) -> Bytecode;
+    fn code_by_hash(&self, code_hash: H256) -> Result<Bytecode, Self::Error>;
     /// Get storage value of address at index.
-    fn storage(&self, address: H160, index: U256) -> U256;
+    fn storage(&self, address: H160, index: U256) -> Result<U256, Self::Error>;
 
     // History related
-    fn block_hash(&self, number: U256) -> H256;
+    fn block_hash(&self, number: U256) -> Result<H256, Self::Error>;
 }
 
-pub struct RefDBWrapper<'a> {
-    pub db: &'a dyn DatabaseRef,
+pub struct RefDBWrapper<'a, Error> {
+    pub db: &'a dyn DatabaseRef<Error = Error>,
 }
 
-impl<'a> RefDBWrapper<'a> {
-    pub fn new(db: &'a dyn DatabaseRef) -> Self {
+impl<'a, Error> RefDBWrapper<'a, Error> {
+    pub fn new(db: &'a dyn DatabaseRef<Error = Error>) -> Self {
         Self { db }
     }
 }
 
-impl<'a> Database for RefDBWrapper<'a> {
-    /// Whether account at address exists.
-    // fn exists(&mut self, address: H160) -> Option<AccountInfo> {
-    //     self.db.exists(address)
-    // }
+impl<'a, Error> Database for RefDBWrapper<'a, Error> {
+    type Error = Error;
     /// Get basic account information.
-    fn basic(&mut self, address: H160) -> AccountInfo {
+    fn basic(&mut self, address: H160) -> Result<Option<AccountInfo>, Self::Error> {
         self.db.basic(address)
     }
     /// Get account code by its hash
-    fn code_by_hash(&mut self, code_hash: H256) -> Bytecode {
+    fn code_by_hash(&mut self, code_hash: H256) -> Result<Bytecode, Self::Error> {
         self.db.code_by_hash(code_hash)
     }
     /// Get storage value of address at index.
-    fn storage(&mut self, address: H160, index: U256) -> U256 {
+    fn storage(&mut self, address: H160, index: U256) -> Result<U256, Self::Error> {
         self.db.storage(address, index)
     }
 
     // History related
-    fn block_hash(&mut self, number: U256) -> H256 {
+    fn block_hash(&mut self, number: U256) -> Result<H256, Self::Error> {
         self.db.block_hash(number)
     }
 }