[1.0] Improve restart production integration test

tests/TestHarness/Node.py

@@ -541,6 +541,9 @@ def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction
         with open(jsonPath, "w") as f:
             json.dump(protocolFeatureJson, f, indent=2)
 
+    def getFinalizerInfo(self):
+       return self.processUrllibRequest("chain", "get_finalizer_info",silentErrors=False, exitOnError=True)
+
     # Require producer_api_plugin
     def createSnapshot(self):
         return self.processUrllibRequest("producer", "create_snapshot")

tests/production_restart.py

@@ -18,9 +18,10 @@
 # Once everything has been confirmed to be working correctly and finality is advancing, cleanly shut down the producer
 # defproducera node but keep the finalizer node of defproducerb running.
 #
-# Then change the finalizer policy (e.g. switch the order of the two finalizers) to get the nodes into a state where
-# they have a pending finalizer policy. At that point restart the producer node defproducera (with stale production
-# enabled so it produces blocks again).
+# Then change the finalizer policy using an unconfigured key in node defproducera to gurantee
+# to get the node stay in a state where it has a pending finalizer policy because the key was
+# not configured. At that point restart the producer node defproducera with new key configured
+# and stale production enabled so it produces blocks again.
 #
 # The correct behavior is for votes from the finalizer node on the newly produced blocks to be accepted by producer
 # node defproducera, QCs to be formed and included in new blocks, and finality to advance.
@@ -64,28 +65,46 @@
     cluster.biosNode.kill(signal.SIGTERM)
     cluster.waitOnClusterSync(blockAdvancing=5)
 
-    node0 = cluster.getNode(0) # producer
-    node1 = cluster.getNode(1) # finalizer
+    producerNode = cluster.getNode(0)
+    finalizerNode = cluster.getNode(1)
 
     Print("Wait for lib to advance")
-    assert node1.waitForLibToAdvance(), "Node1 did not advance LIB"
-    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB"
+    assert finalizerNode.waitForLibToAdvance(), "finalizerNode did not advance LIB"
+    assert producerNode.waitForLibToAdvance(), "producerNode did not advance LIB"
 
     Print("Set finalizers so a pending is in play")
-    assert cluster.setFinalizers([node1, node0], node0), "setfinalizers failed" # switch order
-    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB after setfinalizers"
-    node0.waitForHeadToAdvance() # get additional qc
-
-    Print("Shutdown producer node0")
-    node0.kill(signal.SIGTERM)
-    assert not node0.verifyAlive(), "Node0 did not shutdown"
-
-    Print("Restart producer node0")
-    node0.relaunch(chainArg=" -e ")
+    # Use an unconfigured key for new finalizer policy on producerNode such that
+    # producerNode stays in a state where it has a pending finalizer policy.
+    producerNode.keys[0].blspubkey = "PUB_BLS_JzblSr2sf_UhxQjGxOtHbRCBkHgSB1RG4xUbKKl-fKtUjx6hyOHajnVQT4IvBF4PutlX7JTC14IqIjADlP-3_G2MXRhBlkB57r2u59OCwRQQEDqmVSADf6CoT8zFUXcSgHFw7w" # setFinalizers uses the first key in key list (index 0)
+    producerNode.keys[0].blspop    = "SIG_BLS_Z5fJqFv6DIsHFhBFpkHmL_R48h80zVKQHtB5lrKGOVZTaSQNuVaXD_eHg7HBvKwY6zqgA_vryCLQo5W0Inu6HtLkGL2gYX2UHJjrZJZpfJSKG0ynqAZmyrCglxRLNm8KkFdGGR8oJXf5Yzyu7oautqTPniuKLBvNeQxGJGDOQtHSQ0uP3mD41pWzPFRoi10BUor9MbwUTQ7fO7Of4ZjhVM3IK4JrqX1RBXkDX83Wi9xFzs_fdPIyMqmgEzFgolgUa8XN4Q"
+
+    assert cluster.setFinalizers([producerNode, finalizerNode], producerNode), "setfinalizers failed"
+    assert producerNode.waitForLibToAdvance(), "producerNode did not advance LIB after setfinalizers"
+    # Wait for head to advance twice to make sure pending policy is in place
+    producerNode.waitForHeadToAdvance()
+    producerNode.waitForHeadToAdvance()
+
+    # Check if a pending policy exists
+    finalizerInfo = producerNode.getFinalizerInfo()
+    Print(f"{finalizerInfo}")
+    if (finalizerInfo["payload"]["pending_finalizer_policy"] is not None
+        and finalizerInfo["payload"]["pending_finalizer_policy"]["finalizers"] is not None):
+        Print("pending policy exists")
+    else:
+        Utils.errorExit("pending policy does not exist")
+
+    Print("Shutdown producer producerNode")
+    producerNode.kill(signal.SIGTERM)
+    assert not producerNode.verifyAlive(), "producerNode did not shutdown"
+
+    # Configure the new key (using --signature-provider) and restart producerNode.
+    # LIB should advance
+    Print("Restart producer producerNode")
+    producerNode.relaunch(chainArg=" -e --signature-provider PUB_BLS_JzblSr2sf_UhxQjGxOtHbRCBkHgSB1RG4xUbKKl-fKtUjx6hyOHajnVQT4IvBF4PutlX7JTC14IqIjADlP-3_G2MXRhBlkB57r2u59OCwRQQEDqmVSADf6CoT8zFUXcSgHFw7w=KEY:PVT_BLS_QRxLAVbe2n7RaPWx2wHbur8erqUlAs-V_wXasGhjEA78KlBq")
 
     Print("Verify LIB advances after restart")
-    assert node0.waitForLibToAdvance(), "Node0 did not advance LIB"
-    assert node1.waitForLibToAdvance(), "Node1 did not advance LIB"
+    assert producerNode.waitForLibToAdvance(), "producerNode did not advance LIB"
+    assert finalizerNode.waitForLibToAdvance(), "finalizerNode did not advance LIB"
 
     testSuccessful=True
 finally: