[1.0] Fix issue with last_pending_finalizer_policy_start_timestamp not being set/reflected

libraries/chain/block_header_state.cpp

@@ -301,6 +301,9 @@ void finish_next(const block_header_state& prev,
    // ----------------
    next_header_state.active_finalizer_policy = prev.active_finalizer_policy;
 
+   // will be reset in evaluate_finalizer_policies_for_promotion if needed
+   next_header_state.last_pending_finalizer_policy_start_timestamp = prev.last_pending_finalizer_policy_start_timestamp;
+
    evaluate_finalizer_policies_for_promotion(prev, next_header_state);
 
    next_header_state.last_pending_finalizer_policy_digest = fc::sha256::hash(next_header_state.get_last_pending_finalizer_policy());

libraries/chain/include/eosio/chain/block_header_state.hpp

@@ -208,7 +208,7 @@ FC_REFLECT( eosio::chain::block_header_state, (block_id)(header)
             (activated_protocol_features)(core)(active_finalizer_policy)
             (active_proposer_policy)(latest_proposed_proposer_policy)(latest_pending_proposer_policy)(proposed_finalizer_policies)
             (pending_finalizer_policy)(finalizer_policy_generation)
-            (last_pending_finalizer_policy_digest))
+            (last_pending_finalizer_policy_digest)(last_pending_finalizer_policy_start_timestamp))
 
 FC_REFLECT( eosio::chain::level_3_commitments_t, (reversible_blocks_mroot)(latest_qc_claim_block_num )(latest_qc_claim_finality_digest)(latest_qc_claim_timestamp)(timestamp)(base_digest))
 FC_REFLECT( eosio::chain::level_2_commitments_t, (last_pending_fin_pol_digest)(last_pending_fin_pol_start_timestamp)(l3_commitments_digest) )

tests/CMakeLists.txt

@@ -73,6 +73,8 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/disaster_recovery.py ${CMAKE_CURRENT_
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/disaster_recovery_2.py ${CMAKE_CURRENT_BINARY_DIR}/disaster_recovery_2.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/disaster_recovery_2_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/disaster_recovery_2_test_shape.json COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/disaster_recovery_3.py ${CMAKE_CURRENT_BINARY_DIR}/disaster_recovery_3.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/production_restart.py ${CMAKE_CURRENT_BINARY_DIR}/production_restart.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/production_restart_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/production_restart_test_shape.json COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trx_finality_status_test.py ${CMAKE_CURRENT_BINARY_DIR}/trx_finality_status_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trx_finality_status_forked_test.py ${CMAKE_CURRENT_BINARY_DIR}/trx_finality_status_forked_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugin_http_api_test.py ${CMAKE_CURRENT_BINARY_DIR}/plugin_http_api_test.py COPYONLY)
@@ -159,6 +161,8 @@ add_test(NAME disaster_recovery_2 COMMAND tests/disaster_recovery_2.py -v ${UNSH
 set_property(TEST disaster_recovery_2 PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME disaster_recovery_3 COMMAND tests/disaster_recovery_3.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST disaster_recovery_3 PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME production_restart COMMAND tests/production_restart.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST production_restart PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME ship_reqs_across_svnn_test COMMAND tests/ship_reqs_across_svnn_test.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST ship_reqs_across_svnn_test PROPERTY LABELS nonparallelizable_tests)

tests/production_restart.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import signal
+
+from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness.Node import BlockType
+
+###############################################################
+# production_restart
+#
+# Tests restart of a production node with a pending finalizer policy.
+#
+# Start up a network with two nodes. The first node is a producer node, defproducera that has vote-threads enabled. The
+# second node has a producer (defproducerb) and a single finalizer key configured. Use the bios contract to transition
+# to Savanna consensus while keeping the existing producers and using a finalizer policy with the two finalizers.
+#
+# Once everything has been confirmed to be working correctly and finality is advancing, cleanly shut down the producer
+# defproducera node but keep the finalizer node of defproducerb running.
+#
+# Then change the finalizer policy (e.g. switch the order of the two finalizers) to get the nodes into a state where
+# they have a pending finalizer policy. At that point restart the producer node defproducera (with stale production
+# enabled so it produces blocks again).
+#
+# The correct behavior is for votes from the finalizer node on the newly produced blocks to be accepted by producer
+# node defproducera, QCs to be formed and included in new blocks, and finality to advance.
+#
+# Due to the bug in pre-1.0.0-rc1, we expect that on restart the producer node will reject the votes received by the
+# finalizer node because the producer node will be computing the wrong finality digest.#
+#
+###############################################################
+
+Print=Utils.Print
+errorExit=Utils.errorExit
+
+args=TestHelper.parse_args({"-d","--keep-logs","--dump-error-details","-v","--leave-running","--unshared"})
+pnodes=2
+delay=args.d
+debug=args.v
+prod_count = 1 # per node prod count
+total_nodes=pnodes
+dumpErrorDetails=args.dump_error_details
+
+Utils.Debug=debug
+testSuccessful=False
+
+cluster=Cluster(unshared=args.unshared, keepRunning=args.leave_running, keepLogs=args.keep_logs)
+walletMgr=WalletMgr(True, keepRunning=args.leave_running, keepLogs=args.keep_logs)
+
+try:
+    TestHelper.printSystemInfo("BEGIN")
+
+    cluster.setWalletMgr(walletMgr)
+
+    Print(f'producing nodes: {pnodes}, delay between nodes launch: {delay} second{"s" if delay != 1 else ""}')
+
+    Print("Stand up cluster")
+    # For now do not load system contract as it does not support setfinalizer
+    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, totalProducers=pnodes, delay=delay, loadSystemContract=False,
+                      activateIF=True, topo="./tests/production_restart_test_shape.json") is False:
+        errorExit("Failed to stand up eos cluster.")
+
+    assert cluster.biosNode.getInfo(exitOnError=True)["head_block_producer"] != "eosio", "launch should have waited for production to change"
+    cluster.biosNode.kill(signal.SIGTERM)
+    cluster.waitOnClusterSync(blockAdvancing=5)
+
+    node0 = cluster.getNode(0) # producer
+    node1 = cluster.getNode(1) # finalizer
+
+    Print("Wait for lib to advance")
+    assert node1.waitForLibToAdvance(), "Node1 did not advance LIB"
+    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB"
+
+    Print("Set finalizers so a pending is in play")
+    assert cluster.setFinalizers([node1, node0], node0), "setfinalizers failed" # switch order
+    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB after setfinalizers"
+    node0.waitForHeadToAdvance() # get additional qc
+
+    Print("Shutdown producer node0")
+    node0.kill(signal.SIGTERM)
+    assert not node0.verifyAlive(), "Node0 did not shutdown"
+
+    Print("Restart producer node0")
+    node0.relaunch(chainArg=" -e ")
+
+    Print("Verify LIB advances after restart")
+    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB"
+    assert node1.waitForLibToAdvance(), "Node1 did not advance LIB"
+
+    testSuccessful=True
+finally:
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, dumpErrorDetails=dumpErrorDetails)
+
+exitCode = 0 if testSuccessful else 1
+exit(exitCode)

tests/production_restart_test_shape.json

@@ -0,0 +1,65 @@
+{
+  "name": "testnet_",
+  "ssh_helper": {
+    "ssh_cmd": "/usr/bin/ssh",
+    "scp_cmd": "/usr/bin/scp",
+    "ssh_identity": "",
+    "ssh_args": ""
+  },
+  "nodes": {
+      "bios":{
+        "name": "bios",
+        "keys": [
+          {
+            "privkey":"5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3",
+            "pubkey":"EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"
+          }
+        ],
+        "peers": [],
+        "producers": [
+          "eosio"
+        ],
+        "dont_start": false
+      },
+      "testnet_00":{
+        "name": "testnet_00",
+        "keys": [
+          {
+            "privkey":"5Jf4sTk7vwX1MYpLJ2eQFanVvKYXFqGBrCyANPukuP2BJ5WAAKZ",
+            "pubkey":"EOS58B33q9S7oNkgeFfcoW3VJYu4obfDiqn5RHGE2ige6jVjUhymR",
+            "blspubkey":"PUB_BLS_rYRa_-bT7uLOSAfPIBy6NlXFB0YxwROeSuqHzw6s-1cuK_-GJUKqp20ktyAnsO4ZuHdx3BEPDaLronpnL22MXKWM7bvZnkCfbGCD6OzizQqxXkM9N5z5R-OUA4Ime6cF5YTSFg",
+            "blsprivkey":"PVT_BLS_GQjR0E8Hu8KrsTCvLKnlOCIwQijAj2-5KDizQwF-bAY6pise",
+            "blspop":"SIG_BLS_syFMuifUnX2zQQKr0cuHYzQQjsuPrNG75_z6y8fOyYg_twqMICZ0kT7ObbwIOUsLfXx9PVb4-QLEgUYGSRg1NSfeHGjIGkhea82wa3ayfI8elUEU1MStKbeKpys7xUAQz1PEgwcz5dClq3HyLQmMAjpoL74N_Znf0KiNEVZMte-DLF7x_6sAfp_834LthyYHjZYTmdG7belyzlYHKJb6upnZy9nR_zoKpx9jeTd3tzVhoTCuAN6aFw68D_ItY5cWiY2dhA"
+          }
+        ],
+        "peers": [
+          "bios",
+          "testnet_01"
+        ],
+        "producers": [
+          "defproducera"
+        ],
+        "dont_start": false
+      },
+      "testnet_01":{
+        "name": "testnet_01",
+        "keys": [
+          {
+            "pubkey": "EOS8XH2gKxsef9zxmMHm4vaSvxQUhg7W4GC3nK2KSRxyYrNG5gZFS",
+            "privkey": "5JcoRRhDcgm51dkBrRTmErceTqrYhrq22UnmUjTZToMpH91B9N1",
+            "blspubkey":"PUB_BLS_Wf_O_QeyVhekDXS5q3qBxTyj_qxSrX_uiCY4z8ClpW0X2jrAVgAVHOQ9IR2H40QTWveD8QIGhhSbmSFPa0zFbs5k3yfnjfuuwpA7T1O13_LSdtxT19ehYiE4chZX6SUMJ09JFA",
+            "blsprivkey":"PVT_BLS_1ZLWim0k80ssXswSZp1T3ydHO9U3gLnKKlEBIDy8927XDLLj",
+            "blspop":"SIG_BLS_EL09aI3w-qCgarLM2Z5-T6sisSHBN0J4vMZxtGQklkOcAxgnCaPPXe0roxY4W0gVe2y6T01YrklmT_qZu2tAwqiNrVJcScY8QKvRSeczGBBab1MgnHvaAOuf6bA4JPAELIu2iPWfsS6-oLyLbNP5xtZpMXPHu3yaSJssXNOb5rcVs1KXaIUEagJeAlBBQEcKmFWfeAsJ_R8JDw4i9gSNmROzUjm6LVBpvB7vrnPDPFRA0BQ19H4FED6PtuFPShwJGVz4dg"
+          }
+        ],
+        "peers": [
+          "bios",
+          "testnet_00"
+        ],
+        "producers": [
+          "defproducerb"
+        ],
+        "dont_start": false
+      }
+  }
+}