Merge pull request #479 from ethereum/state_statetest

evmone-statetest tool with JSON test loading

circle.yml

@@ -336,6 +336,35 @@ jobs:
       - upload_coverage:
           flags: consensus
 
+  state-tests:
+    executor: consensus-tests
+    environment:
+      BUILD_TYPE: Coverage
+      CMAKE_OPTIONS: -DCMAKE_CXX_FLAGS=-Og
+    steps:
+      - build
+      - run:
+          # TODO: Merge with download_consensus_tests
+          name: "Download tests"
+          working_directory: ~/tests
+          command: |
+            git clone --depth=50 --single-branch https://github.com/ethereum/tests .
+            git checkout cde14b047d0d2549ed8fd46b4946aed0ec938cbf
+            git submodule init LegacyTests
+            git config -f .gitmodules submodule.LegacyTests.shallow true
+            git submodule update
+      - run:
+          name: "State tests"
+          working_directory: ~/build
+          command: bin/evmone-statetest ~/tests/GeneralStateTests
+      - run:
+          name: "State tests (legacy)"
+          working_directory: ~/build
+          command: bin/evmone-statetest ~/tests/LegacyTests/Constantinople/GeneralStateTests
+      - collect_coverage_gcc
+      - upload_coverage:
+          flags: statetests
+
   gcc-min:
     executor: linux-gcc-min
     steps:
@@ -487,6 +516,7 @@ workflows:
               ignore: /.*/
             tags:
               only: /^v[0-9].*/
+      - state-tests
       - consensus-tests
       - cmake-min
       - gcc-min

test/CMakeLists.txt

@@ -7,6 +7,9 @@ include(${PROJECT_SOURCE_DIR}/evmc/cmake/EVMC.cmake)
 
 set(evmone_private_include_dir ${PROJECT_SOURCE_DIR}/lib)
 
+hunter_add_package(GTest)
+find_package(GTest CONFIG REQUIRED)
+
 hunter_add_package(benchmark)
 find_package(benchmark CONFIG REQUIRED)
 
@@ -15,9 +18,10 @@ add_subdirectory(bench)
 add_subdirectory(integration)
 add_subdirectory(internal_benchmarks)
 add_subdirectory(state)
+add_subdirectory(statetest)
 add_subdirectory(unittests)
 
-set(targets evmone-bench evmone-bench-internal evmone-unittests testutils)
+set(targets evmone-bench evmone-bench-internal evmone-state evmone-statetest evmone-unittests testutils)
 
 if(EVMONE_FUZZING)
     add_subdirectory(fuzzer)

test/state/CMakeLists.txt

@@ -14,4 +14,5 @@ target_sources(
     mpt_hash.hpp
     mpt_hash.cpp
     rlp.hpp
+    state.hpp
 )

test/state/account.hpp

@@ -18,6 +18,9 @@ struct StorageValue
 {
     /// The current value.
     bytes32 current{};
+
+    /// The original value.
+    bytes32 original{};
 };
 
 /// The state account.

test/state/state.hpp

@@ -0,0 +1,59 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "account.hpp"
+#include "hash_utils.hpp"
+#include <cassert>
+#include <optional>
+#include <vector>
+
+namespace evmone::state
+{
+class State
+{
+    std::unordered_map<address, Account> m_accounts;
+
+public:
+    /// Creates new account under the address.
+    Account& create(const address& addr)
+    {
+        const auto r = m_accounts.insert({addr, {}});
+        assert(r.second);
+        return r.first->second;
+    }
+};
+
+struct BlockInfo
+{
+    int64_t number = 0;
+    int64_t timestamp = 0;
+    int64_t gas_limit = 0;
+    address coinbase;
+    bytes32 prev_randao;
+    uint64_t base_fee = 0;
+};
+
+using AccessList = std::vector<std::pair<address, std::vector<bytes32>>>;
+
+struct Transaction
+{
+    enum class Kind
+    {
+        legacy,
+        eip1559
+    };
+
+    Kind kind = Kind::legacy;
+    bytes data;
+    int64_t gas_limit;
+    intx::uint256 max_gas_price;
+    intx::uint256 max_priority_gas_price;
+    address sender;
+    std::optional<address> to;
+    intx::uint256 value;
+    AccessList access_list;
+};
+}  // namespace evmone::state

test/statetest/.clang-tidy

@@ -0,0 +1,3 @@
+InheritParentConfig: true
+Checks: >
+  -clang-analyzer-cplusplus.NewDeleteLeaks

test/statetest/CMakeLists.txt

@@ -0,0 +1,15 @@
+# evmone: Fast Ethereum Virtual Machine implementation
+# Copyright 2022 The evmone Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+hunter_add_package(nlohmann_json)
+find_package(nlohmann_json CONFIG REQUIRED)
+
+add_executable(evmone-statetest)
+target_link_libraries(evmone-statetest PRIVATE evmone evmone::state nlohmann_json::nlohmann_json GTest::gtest)
+target_sources(
+    evmone-statetest PRIVATE
+    statetest.hpp
+    statetest.cpp
+    statetest_loader.cpp
+)

test/statetest/statetest.cpp

@@ -0,0 +1,50 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "statetest.hpp"
+#include <gtest/gtest.h>
+#include <iostream>
+
+namespace
+{
+class StateTest : public testing::Test
+{
+    fs::path m_json_test_file;
+
+public:
+    explicit StateTest(fs::path json_test_file) noexcept
+      : m_json_test_file{std::move(json_test_file)}
+    {}
+
+    void TestBody() final { evmone::test::load_state_test(m_json_test_file); }
+};
+}  // namespace
+
+int main(int argc, char* argv[])
+{
+    testing::InitGoogleTest(&argc, argv);  // Process GoogleTest flags.
+
+    if (argc != 2)
+    {
+        std::cerr << "Missing argument with the path to the tests directory\n";
+        return -1;
+    }
+
+    std::vector<fs::path> test_files;
+    const fs::path root_test_dir{argv[1]};
+    std::copy_if(fs::recursive_directory_iterator{root_test_dir},
+        fs::recursive_directory_iterator{}, std::back_inserter(test_files),
+        [](const fs::directory_entry& entry) {
+            return entry.is_regular_file() && entry.path().extension() == ".json";
+        });
+    std::sort(test_files.begin(), test_files.end());
+    for (const auto& p : test_files)
+    {
+        const auto d = fs::relative(p, root_test_dir);
+        testing::RegisterTest(d.parent_path().string().c_str(), d.stem().string().c_str(), nullptr,
+            nullptr, p.string().c_str(), 0, [p]() -> testing::Test* { return new StateTest(p); });
+    }
+
+    return RUN_ALL_TESTS();
+}

test/statetest/statetest.hpp

@@ -0,0 +1,63 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include "../state/state.hpp"
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace evmone::test
+{
+struct TestMultiTransaction : state::Transaction
+{
+    struct Indexes
+    {
+        size_t input = 0;
+        size_t gas_limit = 0;
+        size_t value = 0;
+    };
+
+    std::vector<state::AccessList> access_lists;
+    std::vector<bytes> inputs;
+    std::vector<int64_t> gas_limits;
+    std::vector<intx::uint256> values;
+
+    [[nodiscard]] Transaction get(const Indexes& indexes) const noexcept
+    {
+        Transaction tx{*this};
+        if (!access_lists.empty())
+            tx.access_list = access_lists.at(indexes.input);
+        tx.data = inputs.at(indexes.input);
+        tx.gas_limit = gas_limits.at(indexes.gas_limit);
+        tx.value = values.at(indexes.value);
+        return tx;
+    }
+};
+
+struct StateTransitionTest
+{
+    struct Case
+    {
+        struct Expectation
+        {
+            TestMultiTransaction::Indexes indexes;
+            hash256 state_hash;
+            hash256 logs_hash;
+            bool exception = false;
+        };
+
+        evmc_revision rev;
+        std::vector<Expectation> expectations;
+    };
+
+    state::State pre_state;
+    state::BlockInfo block;
+    TestMultiTransaction multi_tx;
+    std::vector<Case> cases;
+};
+
+StateTransitionTest load_state_test(const fs::path& test_file);
+
+}  // namespace evmone::test