Merge branch 'main' of github.com:AntelopeIO/leap into gh-698-part2

CMakeLists.txt

@@ -109,7 +109,7 @@ set(Boost_USE_MULTITHREADED      ON)
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 # Most boost deps get implictly picked up via fc, as just about everything links to fc. In addition we pick up
 # the pthread dependency through fc.
-find_package(Boost 1.67 REQUIRED COMPONENTS program_options unit_test_framework system)
+find_package(Boost 1.70 REQUIRED COMPONENTS program_options unit_test_framework system)
 
 if( APPLE AND UNIX )
 # Apple Specific Options Here

README.md

@@ -44,7 +44,7 @@ You will need to build on a [supported operating system](#supported-operating-sy
 
 Requirements to build:
 - C++17 compiler and standard library
-- boost 1.67+
+- boost 1.70+
 - CMake 3.16+
 - LLVM 7 - 11 - for Linux only
   - newer versions do not work

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -120,7 +120,7 @@ Config Options for eosio::chain_plugin:
                                         applied to them (may specify multiple
                                         times)
   --read-mode arg (=head)               Database read mode ("head",
-                                        "irreversible").
+                                        "irreversible", "speculative").
                                         In "head" mode: database contains state
                                         changes up to the head block;
                                         transactions received by the node are
@@ -131,7 +131,14 @@ Config Options for eosio::chain_plugin:
                                         received via the P2P network are not
                                         relayed and transactions cannot be
                                         pushed via the chain API.
-
+                                        In "speculative" mode: (DEPRECATED:
+                                        head mode recommended) database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
+                                        included in the blockchain;
+                                        transactions received by the node are
+                                        relayed if valid.                                        
   --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
   --validation-mode arg (=full)         Chain validation mode ("full" or

docs/01_nodeos/07_concepts/05_storage-and-read-modes.md

@@ -29,6 +29,7 @@ The `nodeos` service can be run in different "read" modes. These modes control h
 
 - `head`: this only includes the side effects of confirmed transactions, this mode processes unconfirmed transactions but does not include them.
 - `irreversible`: this mode also includes confirmed transactions only up to those included in the last irreversible block.
+- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 
 A transaction is considered confirmed when a `nodeos` instance has received, processed, and written it to a block on the blockchain, i.e. it is in the head block or an earlier block.
 
@@ -44,6 +45,16 @@ When `nodeos` is configured to be in irreversible read mode, it will still track
 
 Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
 
+### Speculative Mode ( Deprecated )
+
+Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
+
+Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.
+
+This mode features the lowest latency, but is the least consistent.
+
+In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
+
 ## How To Specify the Read Mode
 
 The mode in which `nodeos` is run can be specified using the `--read-mode` option from the `eosio::chain_plugin`.

libraries/chain/controller.cpp

@@ -270,9 +270,7 @@ struct controller_impl {
          prev = fork_db.root();
       }
 
-      if ( read_mode == db_read_mode::HEAD ) {
-         EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
-      }
+      EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
 
       head = prev;
 
@@ -1635,7 +1633,7 @@ struct controller_impl {
             if ( trx->is_transient() ) {
                // remove trx from pending block by not canceling 'restore'
                trx_context.undo(); // this will happen automatically in destructor, but make it more explicit
-            } else if ( pending->_block_status == controller::block_status::ephemeral ) {
+            } else if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::ephemeral ) {
                // An ephemeral block will never become a full block, but on a producer node the trxs should be saved
                // in the un-applied transaction queue for execution during block production. For a non-producer node
                // save the trxs in the un-applied transaction queue for use during block validation to skip signature

libraries/chain/include/eosio/chain/controller.hpp

@@ -49,7 +49,8 @@ namespace eosio { namespace chain {
 
    enum class db_read_mode {
       HEAD,
-      IRREVERSIBLE
+      IRREVERSIBLE,
+      SPECULATIVE
    };
 
    enum class validation_mode {

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -381,6 +381,10 @@ namespace eosio { namespace chain {
                                     3080007, "Transaction exceeded the current greylisted account network usage limit" )
       FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception,
                                     3080008, "Transaction exceeded the current greylisted account CPU usage limit" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_temporary_failure, resource_exhausted_exception,
+                                    3080009, "Read-only transaction eos-vm-oc compile temporary failure" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception,
+                                    3080010, "Read-only transaction eos-vm-oc compile permanent failure" )
 
       FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
                                     3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )

libraries/chain/include/eosio/chain/wasm_interface.hpp

@@ -61,9 +61,6 @@ namespace eosio { namespace chain {
          //Calls apply or error on a given code
          void apply(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, apply_context& context);
 
-         //Immediately exits currently running wasm. UB is called when no wasm running
-         void exit();
-
          //Returns true if the code is cached
          bool is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const;
 

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -32,7 +32,6 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
       std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
       void init_thread_local_data() override;
 
       friend eosvmoc_instantiated_module;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -37,6 +37,7 @@ using allocator_t = bip::rbtree_best_fit<bip::null_mutex_family, bip::offset_ptr
 
 struct config;
 
+
 class code_cache_base {
    public:
       code_cache_base(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
@@ -46,6 +47,12 @@ class code_cache_base {
 
       void free_code(const digest_type& code_id, const uint8_t& vm_version);
 
+      // get_descriptor_for_code failure reasons
+      enum class get_cd_failure {
+         temporary, // oc compile not done yet, users like read-only trxs can retry
+         permanent  // oc will not start, users should not retry
+      };
+
    protected:
       struct by_hash;
 
@@ -95,7 +102,7 @@ class code_cache_async : public code_cache_base {
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
+      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure);
 
    private:
       std::thread _monitor_reply_thread;

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -46,12 +46,8 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
       std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
-
    private:
       // todo: managing this will get more complicated with sync calls;
-      //       immediately_exit_currently_running_module() should probably
-      //       move from wasm_runtime_interface to wasm_instantiated_module_interface.
       eos_vm_backend_t<Backend>* _bkend = nullptr;  // non owning pointer to allow for immediate exit
 
    template<typename Impl>
@@ -63,8 +59,6 @@ class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
       eos_vm_profile_runtime();
       std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
-
-      void immediately_exit_currently_running_module() override;
 };
 
 }}}}// eosio::chain::webassembly::eos_vm_runtime

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -23,9 +23,6 @@ class wasm_runtime_interface {
       virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
-      //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.
-      virtual void immediately_exit_currently_running_module() = 0;
-
       virtual ~wasm_runtime_interface();
 
       // eosvmoc_runtime needs this

libraries/chain/wasm_interface.cpp

@@ -92,8 +92,9 @@ namespace eosio { namespace chain {
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if(my->eosvmoc) {
          const chain::eosvmoc::code_descriptor* cd = nullptr;
+         chain::eosvmoc::code_cache_base::get_cd_failure failure = chain::eosvmoc::code_cache_base::get_cd_failure::temporary;
          try {
-            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window());
+            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window(), failure);
          }
          catch(...) {
             //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline
@@ -107,15 +108,18 @@ namespace eosio { namespace chain {
             my->eosvmoc->exec->execute(*cd, my->eosvmoc->mem, context);
             return;
          }
+         else if (context.trx_context.is_read_only()) {
+            if (failure == chain::eosvmoc::code_cache_base::get_cd_failure::temporary) {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_temporary_failure, "get_descriptor_for_code failed with temporary failure");
+            } else {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_permanent_failure, "get_descriptor_for_code failed with permanent failure");
+            }
+         }
       }
 #endif
       my->get_instantiated_module(code_hash, vm_type, vm_version, context.trx_context)->apply(context);
    }
 
-   void wasm_interface::exit() {
-      my->runtime_interface->immediately_exit_currently_running_module();
-   }
-
    bool wasm_interface::is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const {
       return my->is_code_cached(code_hash, vm_type, vm_version);
    }

libraries/chain/webassembly/cf_system.cpp

@@ -45,7 +45,8 @@ namespace eosio { namespace chain { namespace webassembly {
       }
    }
 
+   //be aware that EOS VM OC handles eosio_exit internally and this function will not be called by OC
    void interface::eosio_exit( int32_t code ) const {
-      context.control.get_wasm_interface().exit();
+      throw wasm_exit{};
    }
 }}} // ns eosio::chain::webassembly

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -55,9 +55,6 @@ std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }
 
-//never called. EOS VM OC overrides eosio_exit to its own implementation
-void eosvmoc_runtime::immediately_exit_currently_running_module() {}
-
 void eosvmoc_runtime::init_thread_local_data() {
    exec_thread_local = std::make_unique<eosvmoc::executor>(cc);
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -106,7 +106,7 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
 }
 
 
-const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
+const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure) {
    //if there are any outstanding compiles, process the result queue now
    //When app is in write window, all tasks are running sequentially and read-only threads
    //are not running. Safe to update cache entries.
@@ -140,33 +140,44 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_write_window)
+   if(!is_write_window) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    const code_tuple ct = code_tuple{code_id, vm_version};
 
-   if(_blacklist.find(ct) != _blacklist.end())
+   if(_blacklist.find(ct) != _blacklist.end()) {
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
    if(auto it = _outstanding_compiles_and_poison.find(ct); it != _outstanding_compiles_and_poison.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       it->second = false;
       return nullptr;
    }
-   if(_queued_compiles.find(ct) != _queued_compiles.end())
+   if(_queued_compiles.find(ct) != _queued_compiles.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    if(_outstanding_compiles_and_poison.size() >= _threads) {
       _queued_compiles.emplace(ct);
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
    }
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
-   if(!codeobject) //should be impossible right?
+   if(!codeobject) { //should be impossible right?
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
 
    _outstanding_compiles_and_poison.emplace(ct, false);
    std::vector<wrapped_fd> fds_to_pass;
    fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
    write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct }, fds_to_pass);
+   failure = get_cd_failure::temporary; // Compile might not be done yet
    return nullptr;
 }
 

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -233,11 +233,6 @@ class eos_vm_profiling_module : public wasm_instantiated_module_interface {
 template<typename Impl>
 eos_vm_runtime<Impl>::eos_vm_runtime() {}
 
-template<typename Impl>
-void eos_vm_runtime<Impl>::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
 template<typename Impl>
 std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                              const digest_type&, const uint8_t&, const uint8_t&) {
@@ -261,10 +256,6 @@ template class eos_vm_runtime<eosio::vm::jit>;
 
 eos_vm_profile_runtime::eos_vm_profile_runtime() {}
 
-void eos_vm_profile_runtime::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
 std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                                const digest_type&, const uint8_t&, const uint8_t&) {
 

libraries/libfc/include/fc/log/appender.hpp

@@ -3,11 +3,7 @@
 #include <fc/string.hpp>
 #include <memory>
 
-#if BOOST_VERSION >= 106600
 namespace boost { namespace asio { class io_context; typedef io_context io_service; } }
-#else
-namespace boost { namespace asio { class io_service; } }
-#endif
 
 namespace fc {
    class appender;

libraries/libfc/test/network/test_message_buffer.cpp

@@ -7,19 +7,11 @@
 
 namespace {
 size_t mb_size(boost::asio::mutable_buffer& mb) {
-#if BOOST_VERSION >= 106600
    return mb.size();
-#else
-   return boost::asio::detail::buffer_size_helper(mb);
-#endif
 }
 
 void* mb_data(boost::asio::mutable_buffer& mb) {
-#if BOOST_VERSION >= 106600
    return mb.data();
-#else
-   return boost::asio::detail::buffer_cast_helper(mb);
-#endif
 }
 }