From 7f6a10f36d8e5d356b091e6550aace482b92cdbc Mon Sep 17 00:00:00 2001 From: johndebord Date: Tue, 29 Dec 2020 12:45:15 -0500 Subject: [PATCH 1/2] Consolidated Security Fixes for 2.0.9 - Fixes to packed_transaction cache. - Transaction account fail limit refactor. Co-Authored-By: Kevin Heifner --- libraries/chain/controller.cpp | 1 + .../chain/include/eosio/chain/transaction.hpp | 6 ++ plugins/producer_plugin/producer_plugin.cpp | 88 ++++++++++++++++++- 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index a3b48bec848..00b03f7da0c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1733,6 +1733,7 @@ struct controller_impl { if( std::holds_alternative(receipt.trx)) { const auto& pt = std::get(receipt.trx); transaction_metadata_ptr trx_meta_ptr = trx_lookup ? trx_lookup( pt.id() ) : transaction_metadata_ptr{}; + if( trx_meta_ptr && *trx_meta_ptr->packed_trx() != pt ) trx_meta_ptr = nullptr; if( trx_meta_ptr && ( skip_auth_checks || !trx_meta_ptr->recovered_keys().empty() ) ) { trx_metas.emplace_back( std::move( trx_meta_ptr ), recover_keys_future{} ); } else if( skip_auth_checks ) { diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 8875b345b5a..eccafc36cdc 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -157,6 +157,12 @@ namespace eosio { namespace chain { packed_transaction_v0( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); packed_transaction_v0( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + friend bool operator==(const packed_transaction& lhs, const packed_transaction& rhs) { + return std::tie(lhs.signatures, lhs.compression, lhs.packed_context_free_data, lhs.packed_trx) == + std::tie(rhs.signatures, rhs.compression, rhs.packed_context_free_data, rhs.packed_trx); + } + friend bool operator!=(const packed_transaction& lhs, const packed_transaction& rhs) { return !(lhs == rhs); } + uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 8a61a0fe471..eb5b18c031e 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -180,6 +180,7 @@ class producer_plugin_impl : public std::enable_shared_from_this _protocol_features_to_activate; @@ -581,7 +582,7 @@ class producer_plugin_impl : public std::enable_shared_from_this()->default_value( 1024 ), "Maximum size (in MiB) of the incoming transaction queue. Exceeding this value will subjectively drop transaction with resource exhaustion.") + ("disable-api-persisted-trx", bpo::bool_switch()->default_value(false), + "Disable the re-apply of API transactions.") ("producer-threads", bpo::value()->default_value(config::default_controller_thread_pool_size), "Number of worker threads in producer thread pool") ("snapshots-dir", bpo::value()->default_value("snapshots"), @@ -850,6 +853,8 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ my->_incoming_defer_ratio = options.at("incoming-defer-ratio").as(); + my->_disable_persist_until_expired = options.at("disable-api-persisted-trx").as(); + auto thread_pool_size = options.at( "producer-threads" ).as(); EOS_ASSERT( thread_pool_size > 0, plugin_config_exception, "producer-threads ${num} must be greater than 0", ("num", thread_pool_size)); @@ -1713,10 +1718,79 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point return !exhausted; } +namespace { +// track multiple deadline / transaction cpu exceeded exceptions on unapplied transactions +class account_failures { +public: + constexpr static uint32_t max_failures_per_account = 3; + + void add( const account_name& n, int64_t exception_code ) { + if( exception_code == deadline_exception::code_value || exception_code == tx_cpu_usage_exceeded::code_value ) { + auto& fa = failed_accounts[n]; + ++fa.num_failures; + fa.add( exception_code ); + } + } + + // return true if exceeds max_failures_per_account and should be dropped + bool failure_limit( const account_name& n ) { + auto fitr = failed_accounts.find( n ); + if( fitr != failed_accounts.end() && fitr->second.num_failures >= max_failures_per_account ) { + ++fitr->second.num_failures; + return true; + } + return false; + } + + void report() const { + if( _log.is_enabled( fc::log_level::debug ) ) { + for( const auto& e : failed_accounts ) { + std::string reason; + if( e.second.num_failures > max_failures_per_account ) { + reason.clear(); + if( e.second.is_deadline() ) reason += "deadline"; + if( e.second.is_tx_cpu_usage() ) { + if( !reason.empty() ) reason += ", "; + reason += "tx_cpu_usage"; + } + fc_dlog( _log, "Dropped ${n} trxs, account: ${a}, reason: ${r} exceeded", + ("n", e.second.num_failures - max_failures_per_account)("a", e.first)("r", reason) ); + } + } + } + } + +private: + struct account_failure { + enum class ex_fields : uint8_t { + ex_deadline_exception = 1, + ex_tx_cpu_usage_exceeded = 2 + }; + + void add(int64_t exception_code = 0) { + if( exception_code == tx_cpu_usage_exceeded::code_value ) { + ex_flags = set_field( ex_flags, ex_fields::ex_tx_cpu_usage_exceeded ); + } else if( exception_code == deadline_exception::code_value ) { + ex_flags = set_field( ex_flags, ex_fields::ex_deadline_exception ); + } + } + bool is_deadline() const { return has_field( ex_flags, ex_fields::ex_deadline_exception ); } + bool is_tx_cpu_usage() const { return has_field( ex_flags, ex_fields::ex_tx_cpu_usage_exceeded ); } + + uint32_t num_failures = 0; + uint8_t ex_flags = 0; + }; + + std::map failed_accounts; +}; + +} // anonymous namespace + bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadline ) { bool exhausted = false; if( !_unapplied_transactions.empty() ) { + account_failures account_fails; chain::controller& chain = chain_plug->chain(); int num_applied = 0, num_failed = 0, num_processed = 0; auto unapplied_trxs_size = _unapplied_transactions.size(); @@ -1736,11 +1810,16 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin try { auto start = fc::time_point::now(); auto trx_deadline = start + fc::milliseconds( _max_transaction_time_ms ); + if( account_fails.failure_limit( trx->packed_trx()->get_transaction().first_authorizer() ) ) { + ++num_failed; + itr = _unapplied_transactions.erase( itr ); + continue; + } auto prev_billed_cpu_time_us = trx->billed_cpu_time_us; if( prev_billed_cpu_time_us > 0 ) { - auto prev_billed_plus50 = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 50 * config::percent_1 ); - auto trx_dl = start + fc::microseconds( prev_billed_plus50 ); + auto prev_billed_plus100 = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 100 * config::percent_1 ); + auto trx_dl = start + fc::microseconds( prev_billed_plus100 ); if( trx_dl < trx_deadline ) trx_deadline = trx_dl; } bool deadline_is_subjective = false; @@ -1760,10 +1839,12 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin } // don't erase, subjective failure so try again next time } else { + auto failure_code = trace->except->code(); // this failed our configured maximum transaction time, we don't want to replay it fc_dlog( _log, "Failed ${c} trx, prev billed: ${p}us, ran: ${r}us, id: ${id}", ("c", trace->except->code())("p", prev_billed_cpu_time_us) ("r", fc::time_point::now() - start)("id", trx->id()) ); + account_fails.add( trx->packed_trx()->get_transaction().first_authorizer(), failure_code ); ++num_failed; if( itr->next ) itr->next( trace ); itr = _unapplied_transactions.erase( itr ); @@ -1783,6 +1864,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin fc_dlog( _log, "Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}", ("m", num_processed)( "n", unapplied_trxs_size )("applied", num_applied)("failed", num_failed) ); + account_fails.report(); } return !exhausted; } From 5a74daa2d9572fc84b130eed098dbde34f6a3afe Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 4 Jan 2021 09:59:55 -0600 Subject: [PATCH 2/2] Add additional required operator== --- .../chain/include/eosio/chain/transaction.hpp | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index eccafc36cdc..662d5264d61 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -157,12 +157,6 @@ namespace eosio { namespace chain { packed_transaction_v0( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); packed_transaction_v0( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); - friend bool operator==(const packed_transaction& lhs, const packed_transaction& rhs) { - return std::tie(lhs.signatures, lhs.compression, lhs.packed_context_free_data, lhs.packed_trx) == - std::tie(rhs.signatures, rhs.compression, rhs.packed_context_free_data, rhs.packed_trx); - } - friend bool operator!=(const packed_transaction& lhs, const packed_transaction& rhs) { return !(lhs == rhs); } - uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; @@ -217,7 +211,13 @@ namespace eosio { namespace chain { struct none { digest_type digest; + digest_type prunable_digest() const; + + friend bool operator==(const none& lhs, const none& rhs) { + return lhs.digest == rhs.digest; + } + friend bool operator!=(const none& lhs, const none& rhs) { return !(lhs == rhs); } }; using segment_type = std::variant; @@ -225,20 +225,41 @@ namespace eosio { namespace chain { struct partial { std::vector signatures; std::vector context_free_segments; + digest_type prunable_digest() const; + + friend bool operator==(const partial& lhs, const partial& rhs) { + return std::tie( lhs.signatures, lhs.context_free_segments ) == + std::tie( rhs.signatures, rhs.context_free_segments ); + } + friend bool operator!=(const partial& lhs, const partial& rhs) { return !(lhs == rhs); } }; struct full { std::vector signatures; std::vector context_free_segments; + digest_type prunable_digest() const; + + friend bool operator==(const full& lhs, const full& rhs) { + return std::tie( lhs.signatures, lhs.context_free_segments ) == + std::tie( rhs.signatures, rhs.context_free_segments ); + } + friend bool operator!=(const full& lhs, const full& rhs) { return !(lhs == rhs); } }; struct full_legacy { std::vector signatures; bytes packed_context_free_data; vector context_free_segments; + digest_type prunable_digest() const; + + friend bool operator==(const full_legacy& lhs, const full_legacy& rhs) { + return std::tie( lhs.signatures, lhs.packed_context_free_data, lhs.context_free_segments ) == + std::tie( rhs.signatures, rhs.packed_context_free_data, rhs.context_free_segments ); + } + friend bool operator!=(const full_legacy& lhs, const full_legacy& rhs) { return !(lhs == rhs); } }; using prunable_data_t = std::variant< full_legacy, @@ -246,6 +267,11 @@ namespace eosio { namespace chain { partial, full >; + friend bool operator==(const prunable_data_type& lhs, const prunable_data_type& rhs) { + return lhs.prunable_data == rhs.prunable_data; + } + friend bool operator!=(const prunable_data_type& lhs, const prunable_data_type& rhs) { return !(lhs == rhs); } + prunable_data_type prune_all() const; digest_type digest() const; @@ -271,6 +297,12 @@ namespace eosio { namespace chain { packed_transaction_v0_ptr to_packed_transaction_v0() const; + friend bool operator==(const packed_transaction& lhs, const packed_transaction& rhs) { + return std::tie(lhs.compression, lhs.prunable_data, lhs.packed_trx) == + std::tie(rhs.compression, rhs.prunable_data, rhs.packed_trx); + } + friend bool operator!=(const packed_transaction& lhs, const packed_transaction& rhs) { return !(lhs == rhs); } + uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; uint32_t get_estimated_size()const { return estimated_size; }