Merge branch 'hotstuff_integration' of github.com:AntelopeIO/leap int…

…o gh_2028

.github/workflows/build.yaml

@@ -93,7 +93,7 @@ jobs:
         with:
           submodules: recursive
       - name: Download builddir
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.platform}}-build
       - name: Build packages
@@ -112,17 +112,19 @@ jobs:
         run: |
           python3 -c "from TestHarness import Cluster"
       - name: Upload dev package
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: matrix.platform != 'reproducible'
         with:
           name: leap-dev-${{matrix.platform}}-amd64
           path: build/leap-dev*.deb
+          compression-level: 0
       - name: Upload leap package
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: matrix.platform == 'reproducible'
         with:
           name: leap-deb-amd64
           path: build/leap_*.deb
+          compression-level: 0
 
   tests:
     name: Tests (${{matrix.cfg.name}})
@@ -142,7 +144,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Download builddir
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.cfg.builddir}}-build
       - name: Run Parallel Tests
@@ -153,12 +155,13 @@ jobs:
           cd build
           ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420
       - name: Upload core files from failed tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: ${{matrix.cfg.name}}-tests-logs
           if-no-files-found: ignore
           path: /cores
+          compression-level: 0
       - name: Check CPU Features
         run: awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo
 
@@ -177,7 +180,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Download builddir
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.cfg.builddir}}-build
       - name: Run tests in parallel containers
@@ -192,13 +195,14 @@ jobs:
         run: docker run --mount type=bind,source=/var/lib/systemd/coredump,target=/cores alpine sh -c 'tar -C /cores/ -c .' | tar x
         if: failure()
       - name: Upload logs from failed tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: ${{matrix.cfg.name}}-np-logs
           path: |
             *-logs.tar.gz
             core*.zst
+          compression-level: 0
 
   lr-tests:
     name: LR Tests (${{matrix.cfg.name}})
@@ -215,7 +219,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Download builddir
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.cfg.builddir}}-build
       - name: Run tests in parallel containers
@@ -230,13 +234,14 @@ jobs:
         run: docker run --mount type=bind,source=/var/lib/systemd/coredump,target=/cores alpine sh -c 'tar -C /cores/ -c .' | tar x
         if: failure()
       - name: Upload logs from failed tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: ${{matrix.cfg.name}}-lr-logs
           path: |
             *-logs.tar.gz
             core*.zst
+          compression-level: 0
 
   libtester-tests:
     name: libtester tests
@@ -265,7 +270,7 @@ jobs:
           submodules: recursive
       - if: ${{ matrix.test != 'deb-install' }}
         name: Download leap builddir
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{matrix.platform}}-build
       - if: ${{ matrix.test != 'deb-install' }}
@@ -287,7 +292,7 @@ jobs:
           rm -r *
       - if: ${{ matrix.test == 'deb-install' }}
         name: Download leap-dev
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: leap-dev-${{matrix.platform}}-amd64
       - if: ${{ matrix.test == 'deb-install' }}

.github/workflows/build_base.yaml

@@ -42,7 +42,8 @@ jobs:
             cmake --build build
             tar -pc --exclude "*.o" build | zstd --long -T0 -9 > build.tar.zst
         - name: Upload builddir
-          uses: AntelopeIO/upload-artifact-large-chunks-action@v1
+          uses: actions/upload-artifact@v4
           with:
             name: ${{matrix.platform}}-build
-            path: build.tar.zst
+            path: build.tar.zst
+            compression-level: 0

libraries/chain/authorization_manager.cpp

@@ -479,7 +479,12 @@ namespace eosio { namespace chain {
 
       auto effective_provided_delay =  (provided_delay >= delay_max_limit) ? fc::microseconds::maximum() : provided_delay;
 
-      auto checker = make_auth_checker( [&](const permission_level& p){ return get_permission(p).auth; },
+      auto checker = make_auth_checker( [&](const permission_level& p) -> const shared_authority* {
+                                          if(const permission_object* po = find_permission(p))
+                                             return &po->auth;
+                                          else
+                                             return nullptr;
+                                        },
                                         _control.get_global_properties().configuration.max_authority_depth,
                                         provided_keys,
                                         provided_permissions,
@@ -580,7 +585,12 @@ namespace eosio { namespace chain {
 
       auto delay_max_limit = fc::seconds( _control.get_global_properties().configuration.max_transaction_delay );
 
-      auto checker = make_auth_checker( [&](const permission_level& p){ return get_permission(p).auth; },
+      auto checker = make_auth_checker( [&](const permission_level& p) -> const shared_authority* {
+                                          if(const permission_object* po = find_permission(p))
+                                             return &po->auth;
+                                          else
+                                             return nullptr;
+                                        },
                                         _control.get_global_properties().configuration.max_authority_depth,
                                         provided_keys,
                                         provided_permissions,
@@ -611,7 +621,12 @@ namespace eosio { namespace chain {
                                                                        fc::microseconds provided_delay
                                                                      )const
    {
-      auto checker = make_auth_checker( [&](const permission_level& p){ return get_permission(p).auth; },
+      auto checker = make_auth_checker( [&](const permission_level& p) -> const shared_authority* {
+                                          if(const permission_object* po = find_permission(p))
+                                             return &po->auth;
+                                          else
+                                             return nullptr;
+                                        },
                                         _control.get_global_properties().configuration.max_authority_depth,
                                         candidate_keys,
                                         {},

libraries/chain/controller.cpp

@@ -815,6 +815,7 @@ struct controller_impl {
    deep_mind_handler*              deep_mind_logger = nullptr;
    bool                            okay_to_print_integrity_hash_on_stop = false;
    bls_key_map_t                   node_finalizer_keys;
+   std::atomic<bool>               writing_snapshot = false;
 
    thread_local static platform_timer timer; // a copy for main thread and each read-only thread
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
@@ -4299,7 +4300,15 @@ fc::sha256 controller::calculate_integrity_hash() { try {
 
 void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) {
    EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" );
-   return my->add_to_snapshot(snapshot);
+   my->writing_snapshot.store(true, std::memory_order_release);
+   fc::scoped_exit<std::function<void()>> e = [&] {
+      my->writing_snapshot.store(false, std::memory_order_release);
+   };
+   my->add_to_snapshot(snapshot);
+}
+
+bool controller::is_writing_snapshot() const {
+   return my->writing_snapshot.load(std::memory_order_acquire);
 }
 
 int64_t controller::set_proposed_producers( vector<producer_authority> producers ) {

libraries/chain/include/eosio/chain/authority_checker.hpp

@@ -28,8 +28,8 @@ namespace detail {
     * then determine whether that list of keys is sufficient to satisfy the authority. This class takes a list of keys and
     * provides the @ref satisfied method to determine whether that list of keys satisfies a provided authority.
     *
-    * @tparam F A callable which takes a single argument of type @ref AccountPermission and returns the corresponding
-    * authority
+    * @tparam F A callable which takes a single argument of type @ref AccountPermission and returns a pointer to
+    * the corresponding authority if it exists, or a nullptr.
     */
    template<typename PermissionToAuthorityFunc>
    class authority_checker {
@@ -225,19 +225,18 @@ namespace detail {
                      bool r = false;
                      typename permission_cache_type::iterator itr = cached_permissions.end();
 
-                     bool propagate_error = false;
+                     std::invoke_result_t<decltype(checker.permission_to_authority), const permission_level> auth = nullptr;
                      try {
-                        auto&& auth = checker.permission_to_authority( permission.permission );
-                        propagate_error = true;
-                        auto res = cached_permissions.emplace( permission.permission, being_evaluated );
-                        itr = res.first;
-                        r = checker.satisfied( std::forward<decltype(auth)>(auth), cached_permissions, recursion_depth + 1 );
-                     } catch( const permission_query_exception& ) {
-                        if( propagate_error )
-                           throw;
-                        else
-                           return total_weight; // if the permission doesn't exist, continue without it
+                        auth = checker.permission_to_authority( permission.permission );
                      }
+                     catch( const permission_query_exception& ) {}
+                     //permission was either invalid (threw permission_query_exception), or wasn't found
+                     if(!auth)
+                        return total_weight;
+
+                     auto res = cached_permissions.emplace( permission.permission, being_evaluated );
+                     itr = res.first;
+                     r = checker.satisfied( *auth, cached_permissions, recursion_depth + 1 );
 
                      if( r ) {
                         total_weight += permission.weight;

libraries/chain/include/eosio/chain/controller.hpp

@@ -289,6 +289,8 @@ namespace eosio::chain {
 
          fc::sha256 calculate_integrity_hash();
          void write_snapshot( const snapshot_writer_ptr& snapshot );
+         // thread-safe
+         bool is_writing_snapshot()const;
 
          bool sender_avoids_whitelist_blacklist_enforcement( account_name sender )const;
          void check_actor_list( const flat_set<account_name>& actors )const;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp

@@ -5,7 +5,7 @@
 #include <vector>
 #include <string>
 
-#include <fc/reflect/reflect.hpp>
+#include <fc/io/raw.hpp>
 
 #include <sys/resource.h>
 
@@ -25,6 +25,34 @@ struct config {
    std::optional<size_t>   generated_code_size_limit {16u*1024u*1024u};
 };
 
-}}}
+//work around unexpected std::optional behavior
+template <typename DS>
+inline DS& operator>>(DS& ds, eosio::chain::eosvmoc::config& cfg) {
+   fc::raw::pack(ds, cfg.cache_size);
+   fc::raw::pack(ds, cfg.threads);
+
+   auto better_optional_unpack = [&]<typename T>(std::optional<T>& t) {
+      bool b; fc::raw::unpack( ds, b );
+      if(b) { t = T(); fc::raw::unpack( ds, *t ); }
+      else { t.reset(); }
+   };
+   better_optional_unpack(cfg.cpu_limit);
+   better_optional_unpack(cfg.vm_limit);
+   better_optional_unpack(cfg.stack_size_limit);
+   better_optional_unpack(cfg.generated_code_size_limit);
+
+   return ds;
+}
+
+template <typename DS>
+inline DS& operator<<(DS& ds, const eosio::chain::eosvmoc::config& cfg) {
+   fc::raw::pack(ds, cfg.cache_size);
+   fc::raw::pack(ds, cfg.threads);
+   fc::raw::pack(ds, cfg.cpu_limit);
+   fc::raw::pack(ds, cfg.vm_limit);
+   fc::raw::pack(ds, cfg.stack_size_limit);
+   fc::raw::pack(ds, cfg.generated_code_size_limit);
+   return ds;
+}
 
-FC_REFLECT(eosio::chain::eosvmoc::config, (cache_size)(threads)(cpu_limit)(vm_limit)(stack_size_limit)(generated_code_size_limit))
+}}}

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_helpers.hpp

@@ -1,14 +1,12 @@
 #pragma once
 
 #include <eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp>
+#include <eosio/chain/webassembly/eos-vm-oc/memfd_helpers.hpp>
 
 #include <boost/asio/local/datagram_protocol.hpp>
 
 #include <vector>
 
-#include <sys/syscall.h>
-#include <linux/memfd.h>
-
 namespace eosio { namespace chain { namespace eosvmoc {
 
 class wrapped_fd {
@@ -54,7 +52,7 @@ bool write_message_with_fds(int fd_to_send_to, const eosvmoc_message& message, c
 
 template<typename T>
 wrapped_fd memfd_for_bytearray(const T& bytes) {
-   int fd = syscall(SYS_memfd_create, "eosvmoc_code", MFD_CLOEXEC);
+   int fd = exec_sealed_memfd_create("eosvmoc_code");
    FC_ASSERT(fd >= 0, "Failed to create memfd");
    FC_ASSERT(ftruncate(fd, bytes.size()) == 0, "failed to grow memfd");
    if(bytes.size()) {

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memfd_helpers.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <linux/memfd.h>
+#include <sys/mman.h>
+
+namespace eosio::chain::eosvmoc {
+
+// added in glibc 2.38
+#ifndef MFD_NOEXEC_SEAL
+#define MFD_NOEXEC_SEAL 8U
+#endif
+
+inline int exec_sealed_memfd_create(const char* name) {
+   //kernels 6.3 through 6.6 by default warn when neither MFD_NOEXEC_SEAL nor MFD_EXEC are passed; optionally 6.3+
+   // may enforce MFD_NOEXEC_SEAL. Prior to 6.3 these flags will EINVAL.
+   if(int ret = memfd_create(name, MFD_CLOEXEC | MFD_NOEXEC_SEAL); ret >= 0 || errno != EINVAL)
+      return ret;
+   return memfd_create(name, MFD_CLOEXEC);
+}
+
+}

libraries/chain/snapshot.cpp

@@ -153,13 +153,7 @@ void ostream_snapshot_writer::write_start_section( const std::string& section_na
 }
 
 void ostream_snapshot_writer::write_row( const detail::abstract_snapshot_row_writer& row_writer ) {
-   auto restore = snapshot.tellp();
-   try {
-      row_writer.write(snapshot);
-   } catch (...) {
-      snapshot.seekp(restore);
-      throw;
-   }
+   row_writer.write(snapshot);
    row_count++;
 }
 

libraries/chain/webassembly/runtimes/eos-vm-oc/memory.cpp

@@ -1,20 +1,19 @@
 #include <eosio/chain/webassembly/eos-vm-oc/memory.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/intrinsic.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp>
+#include <eosio/chain/webassembly/eos-vm-oc/memfd_helpers.hpp>
 
 #include <fc/scoped_exit.hpp>
 
 #include <unistd.h>
-#include <sys/syscall.h>
 #include <sys/mman.h>
-#include <linux/memfd.h>
 
 namespace eosio { namespace chain { namespace eosvmoc {
 
 memory::memory(uint64_t sliced_pages) {
    uint64_t number_slices = sliced_pages + 1;
    uint64_t wasm_memory_size = sliced_pages * wasm_constraints::wasm_page_size;
-   int fd = syscall(SYS_memfd_create, "eosvmoc_mem", MFD_CLOEXEC);
+   int fd = exec_sealed_memfd_create("eosvmoc_mem");
    FC_ASSERT(fd >= 0, "Failed to create memory memfd");
    auto cleanup_fd = fc::make_scoped_exit([&fd](){close(fd);});
    int ret = ftruncate(fd, wasm_memory_size+memory_prologue_size);