IF: Use finalizer diffs in instant_finality_extension

libraries/chain/block_header_state.cpp

@@ -67,6 +67,28 @@ const vector<digest_type>& block_header_state::get_new_protocol_feature_activati
    return detail::get_new_protocol_feature_activations(header_exts);
 }
 
+// The last proposed finalizer policy if none proposed or pending is the active finalizer policy
+const finalizer_policy& block_header_state::get_last_proposed_finalizer_policy() const {
+   if (!finalizer_policies.empty()) {
+      for (auto ritr = finalizer_policies.rbegin(); ritr != finalizer_policies.rend(); ++ritr) {
+         if (ritr->second.state == finalizer_policy_tracker::state_t::proposed)
+            return *ritr->second.policy;
+      }
+      return *finalizer_policies.rbegin()->second.policy;
+   }
+   return *active_finalizer_policy;
+}
+
+finalizer_policy_diff block_header_state::calculate_finalizer_policy_diff(const finalizer_policy& new_policy) const {
+   return get_last_proposed_finalizer_policy().create_diff(new_policy);
+}
+
+finalizer_policy block_header_state::calculate_finalizer_policy(const finalizer_policy_diff& diff) const {
+   finalizer_policy result = get_last_proposed_finalizer_policy();
+   result.apply_diff(diff);
+   return result;
+}
+
 // -------------------------------------------------------------------------------------------------
 // `finish_next` updates the next `block_header_state` according to the contents of the
 // header extensions (either new protocol_features or instant_finality_extension) applicable to this
@@ -156,20 +178,21 @@ void finish_next(const block_header_state& prev,
       }
    }
 
-   if (if_ext.new_finalizer_policy) {
+   if (if_ext.new_finalizer_policy_diff) {
+      finalizer_policy new_finalizer_policy = prev.calculate_finalizer_policy(*if_ext.new_finalizer_policy_diff);
+
       // a new `finalizer_policy` was proposed in the previous block, and is present in the previous
       // block's header extensions.
       // Add this new proposal to the `finalizer_policies` multimap which tracks the in-flight proposals,
       // increment the generation number, and log that proposal (debug level).
       // ------------------------------------------------------------------------------------------------
-      dlog("New finalizer policy proposed in block ${id}: ${pol}",
-           ("id", prev.block_id)("pol", *if_ext.new_finalizer_policy));
-      next_header_state.finalizer_policy_generation = if_ext.new_finalizer_policy->generation;
+      dlog("New finalizer policy proposed in block ${id}..: ${pol}",
+           ("id", prev.block_id.str().substr(8,16))("pol", new_finalizer_policy));
+      next_header_state.finalizer_policy_generation = new_finalizer_policy.generation;
       next_header_state.finalizer_policies.emplace(
          next_header_state.block_num(),
          finalizer_policy_tracker{finalizer_policy_tracker::state_t::proposed,
-                                  std::make_shared<finalizer_policy>(std::move(*if_ext.new_finalizer_policy))});
-
+                                  std::make_shared<finalizer_policy>(std::move(new_finalizer_policy))});
    } else {
       next_header_state.finalizer_policy_generation = prev.finalizer_policy_generation;
    }
@@ -205,8 +228,12 @@ block_header_state block_header_state::next(block_header_state_input& input) con
 
    // finality extension
    // ------------------
+   std::optional<finalizer_policy_diff> new_finalizer_policy_diff;
+   if (input.new_finalizer_policy) {
+      new_finalizer_policy_diff = calculate_finalizer_policy_diff(*input.new_finalizer_policy);
+   }
    instant_finality_extension new_if_ext { input.most_recent_ancestor_with_qc,
-                                           std::move(input.new_finalizer_policy),
+                                           std::move(new_finalizer_policy_diff),
                                            std::move(input.new_proposer_policy) };
 
    uint16_t if_ext_id = instant_finality_extension::extension_id();

libraries/chain/block_header_state_legacy.cpp

@@ -220,8 +220,10 @@ namespace eosio::chain {
          // set current block_num as qc_claim.last_qc_block_num in the IF extension
          qc_claim_t initial_if_claim { .block_num = block_num,
                                        .is_strong_qc = false };
+         finalizer_policy no_policy;
+         auto new_fin_policy_diff = no_policy.create_diff(*new_finalizer_policy);
          emplace_extension(h.header_extensions, instant_finality_extension::extension_id(),
-                           fc::raw::pack(instant_finality_extension{ initial_if_claim, std::move(new_finalizer_policy), {} }));
+                           fc::raw::pack(instant_finality_extension{ initial_if_claim, std::move(new_fin_policy_diff), {} }));
       } else if (qc_claim) {
          emplace_extension(h.header_extensions, instant_finality_extension::extension_id(),
                            fc::raw::pack(instant_finality_extension{ *qc_claim, {}, {} }));

libraries/chain/block_state.cpp

@@ -72,8 +72,9 @@ block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& b
 
    assert(bsp.block->contains_header_extension(instant_finality_extension::extension_id())); // required by transition mechanism
    instant_finality_extension if_ext = bsp.block->extract_header_extension<instant_finality_extension>();
-   assert(if_ext.new_finalizer_policy); // required by transition mechanism
-   result.active_finalizer_policy = std::make_shared<finalizer_policy>(*if_ext.new_finalizer_policy);
+   assert(if_ext.new_finalizer_policy_diff); // required by transition mechanism
+   result.active_finalizer_policy = std::make_shared<finalizer_policy>();
+   result.active_finalizer_policy->apply_diff(std::move(*if_ext.new_finalizer_policy_diff));
    result.active_proposer_policy = std::make_shared<proposer_policy>();
    result.active_proposer_policy->active_time = bsp.timestamp();
    result.active_proposer_policy->proposer_schedule = bsp.active_schedule;

libraries/chain/include/eosio/chain/block_header_state.hpp

@@ -46,8 +46,8 @@ struct finality_digest_data_v1 {
 // ------------------------------------------------------------------------------------------
 struct finalizer_policy_tracker {
    enum class state_t { proposed = 0, pending };
-   state_t               state;
-   finalizer_policy_ptr  policy;
+   state_t                    state;
+   finalizer_policy_ptr       policy;
 };
 
 struct building_block_input {
@@ -129,6 +129,10 @@ struct block_header_state {
 
    const vector<digest_type>& get_new_protocol_feature_activations() const;
    const producer_authority& get_scheduled_producer(block_timestamp_type t) const;
+
+   const finalizer_policy& get_last_proposed_finalizer_policy() const;
+   finalizer_policy_diff calculate_finalizer_policy_diff(const finalizer_policy& new_policy) const;
+   finalizer_policy calculate_finalizer_policy(const finalizer_policy_diff& diff) const;
 };
 
 using block_header_state_ptr = std::shared_ptr<block_header_state>;

libraries/chain/include/eosio/chain/config.hpp

@@ -134,7 +134,7 @@ static_assert(maximum_tracked_dpos_confirmations >= ((max_producers * 2 / 3) + 1
 /**
  * Maximum number of finalizers in the finalizer set
  */
-const static size_t max_finalizers = 64*1024;
+const static size_t max_finalizers = 64*1024; // largest allowed finalizer policy diff
 const static size_t max_finalizer_description_size = 256;
 
 /**

libraries/chain/include/eosio/chain/finality/finalizer_policy.hpp

@@ -2,13 +2,38 @@
 
 #include <eosio/chain/types.hpp>
 #include <eosio/chain/finality/finalizer_authority.hpp>
+#include <fc/container/ordered_diff.hpp>
 
 namespace eosio::chain {
 
+   static_assert(std::numeric_limits<uint16_t>::max() >= config::max_finalizers - 1);
+   using finalizers_differ = fc::ordered_diff<finalizer_authority, uint16_t>;
+   using finalizers_diff_t = finalizers_differ::diff_result;
+
+   struct finalizer_policy_diff {
+      uint32_t                         generation = 0; ///< sequentially incrementing version number
+      uint64_t                         threshold = 0;  ///< vote weight threshold to finalize blocks
+      finalizers_diff_t                finalizers_diff;
+   };
+
    struct finalizer_policy {
       uint32_t                         generation = 0; ///< sequentially incrementing version number
       uint64_t                         threshold = 0;  ///< vote weight threshold to finalize blocks
-      std::vector<finalizer_authority> finalizers; ///< Instant Finality voter set
+      std::vector<finalizer_authority> finalizers;     ///< Instant Finality voter set
+
+      finalizer_policy_diff create_diff(const finalizer_policy& target) const {
+         return {.generation = target.generation,
+                 .threshold = target.threshold,
+                 .finalizers_diff = finalizers_differ::diff(finalizers, target.finalizers)};
+      }
+
+      template <typename X>
+      requires std::same_as<std::decay_t<X>, finalizer_policy_diff>
+      void apply_diff(X&& diff) {
+         generation = diff.generation;
+         threshold = diff.threshold;
+         finalizers = finalizers_differ::apply_diff(std::move(finalizers), std::forward<X>(diff).finalizers_diff);
+      }
 
       // max accumulated weak weight before becoming weak_final
       uint64_t max_weak_sum_before_weak_final() const {
@@ -23,7 +48,10 @@ namespace eosio::chain {
    };
 
    using finalizer_policy_ptr = std::shared_ptr<finalizer_policy>;
+   using finalizer_policy_diff_ptr = std::shared_ptr<finalizer_policy_diff>;
 
 } /// eosio::chain
 
 FC_REFLECT( eosio::chain::finalizer_policy, (generation)(threshold)(finalizers) )
+FC_REFLECT( eosio::chain::finalizers_diff_t, (remove_indexes)(insert_indexes) )
+FC_REFLECT( eosio::chain::finalizer_policy_diff, (generation)(threshold)(finalizers_diff) )

libraries/chain/include/eosio/chain/finality/instant_finality_extension.hpp

@@ -12,10 +12,10 @@ struct instant_finality_extension : fc::reflect_init {
 
    instant_finality_extension() = default;
    instant_finality_extension(qc_claim_t qc_claim,
-                              std::optional<finalizer_policy> new_finalizer_policy,
+                              std::optional<finalizer_policy_diff> new_finalizer_policy_diff,
                               std::shared_ptr<proposer_policy> new_proposer_policy) :
       qc_claim(qc_claim),
-      new_finalizer_policy(std::move(new_finalizer_policy)),
+      new_finalizer_policy_diff(std::move(new_finalizer_policy_diff)),
       new_proposer_policy(std::move(new_proposer_policy))
    {}
 
@@ -25,11 +25,11 @@ struct instant_finality_extension : fc::reflect_init {
       static_assert( extension_id() == 2, "instant_finality_extension extension id must be 2" );
    }
 
-   qc_claim_t                         qc_claim;
-   std::optional<finalizer_policy>    new_finalizer_policy;
-   std::shared_ptr<proposer_policy>   new_proposer_policy;
+   qc_claim_t                              qc_claim;
+   std::optional<finalizer_policy_diff>    new_finalizer_policy_diff;
+   std::shared_ptr<proposer_policy>        new_proposer_policy;
 };
 
 } /// eosio::chain
 
-FC_REFLECT( eosio::chain::instant_finality_extension, (qc_claim)(new_finalizer_policy)(new_proposer_policy) )
+FC_REFLECT( eosio::chain::instant_finality_extension, (qc_claim)(new_finalizer_policy_diff)(new_proposer_policy) )

libraries/libfc/include/fc/container/ordered_diff.hpp

@@ -0,0 +1,113 @@
+#pragma once
+
+#include <vector>
+#include <utility>
+
+namespace fc {
+
+/**
+ * @class ordered_diff
+ * @brief Provides ability to generate and apply diff of containers of type T
+ *
+ * Example use:
+ *    std::vector<char> source = { 'a', 'b', 'f', 'c', 'd' };
+ *    std::vector<char> target = { 'b', 'f', 'c', 'd', 'e', 'h' };
+ *    ordered_diff<char>::diff_result diff = ordered_diff<char>::diff(source, target);
+ *    ordered_diff<char>::apply_diff(source, std::move(diff));
+ *    assert(source == target);
+ *
+ * @param T type stored in Container, must provide ==
+ * @param SizeType numeric type used for index into diff_result, for non-unique Containers a larger type may be required
+ * @param Container container type for ordered diff and for diff_result
+ */
+template <typename T, typename SizeType = size_t, template<typename Y, typename...> typename Container = std::vector>
+requires std::equality_comparable<T> && std::random_access_iterator<typename Container<T>::iterator>
+class ordered_diff {
+public:
+   struct diff_result {
+      Container<SizeType>                remove_indexes;
+      Container<std::pair<SizeType, T>>  insert_indexes;
+   };
+
+   /// Generate diff_result that when `apply_diff(source, diff_result)` will modify source to be equal to target.
+   static diff_result diff(const Container<T>& source, const Container<T>& target) {
+      size_t s = 0;
+      size_t t = 0;
+
+      diff_result result;
+      while (s < source.size() || t < target.size()) {
+         if (s < source.size() && t < target.size()) {
+            if (source[s] == target[t]) {
+               // nothing to do, skip over
+               assert(s <= std::numeric_limits<SizeType>::max());
+               assert(t <= std::numeric_limits<SizeType>::max());
+               ++s;
+               ++t;
+            } else { // not equal
+               if (s == source.size() - 1 && t == target.size() - 1) {
+                  // both at end, insert target and remove source
+                  assert(s <= std::numeric_limits<SizeType>::max());
+                  assert(t <= std::numeric_limits<SizeType>::max());
+                  result.remove_indexes.push_back(s);
+                  result.insert_indexes.emplace_back(t, target[t]);
+                  ++s;
+                  ++t;
+               } else if (s + 1 < source.size() && t + 1 < target.size() && source[s + 1] == target[t + 1]) {
+                  // misalignment, but next value equal, insert and remove
+                  assert(s <= std::numeric_limits<SizeType>::max());
+                  assert(t <= std::numeric_limits<SizeType>::max());
+                  result.remove_indexes.push_back(s);
+                  result.insert_indexes.emplace_back(t, target[t]);
+                  ++s;
+                  ++t;
+               } else if (t + 1 < target.size() && source[s] == target[t + 1]) {
+                  // source equals next target, insert current target
+                  assert(t <= std::numeric_limits<SizeType>::max());
+                  result.insert_indexes.emplace_back(t, target[t]);
+                  ++t;
+               } else { // source[s + 1] == target[t]
+                  // target matches next source, remove current source
+                  assert(s <= std::numeric_limits<SizeType>::max());
+                  result.remove_indexes.push_back(s);
+                  ++s;
+               }
+            }
+         } else if (s < source.size()) {
+            // remove extra in source
+            assert(s <= std::numeric_limits<SizeType>::max());
+            result.remove_indexes.push_back(s);
+            ++s;
+         } else if (t < target.size()) {
+            // insert extra in target
+            assert(t <= std::numeric_limits<SizeType>::max());
+            result.insert_indexes.emplace_back(t, target[t]);
+            ++t;
+         }
+      }
+
+      return result;
+   }
+
+   /// @param diff the diff_result created from diff(source, target), apply_diff(std::move(source), diff_result) => target
+   /// @param container the source of diff(source, target) to modify using the diff_result to produce original target
+   /// @return the modified container now equal to original target
+   template <typename X>
+   requires std::same_as<std::decay_t<X>, diff_result>
+   static Container<T> apply_diff(Container<T>&& container, X&& diff) {
+      // Remove from the source based on diff.remove_indexes
+      std::ptrdiff_t offset = 0;
+      for (SizeType index : diff.remove_indexes) {
+         container.erase(container.begin() + index + offset);
+         --offset;
+      }
+
+      // Insert into the source based on diff.insert_indexes
+      for (auto& [index, value] : diff.insert_indexes) {
+         container.insert(container.begin() + index, std::move(value));
+      }
+      return container;
+   }
+
+}; // class ordered_diff
+
+} // namespace fc

libraries/libfc/test/CMakeLists.txt

@@ -19,6 +19,7 @@ add_executable( test_fc
         test_base64.cpp
         test_escape_str.cpp
         test_bls.cpp
+        test_ordered_diff.cpp
         main.cpp
         )
 target_link_libraries( test_fc fc )