IF: Add IF activation to integration test framework

libraries/chain/hotstuff/chain_pacemaker.cpp

@@ -169,6 +169,7 @@ namespace eosio::chain {
          if (ext) {
             std::scoped_lock g( _chain_state_mutex );
             if (_active_finalizer_policy.generation == 0) {
+               ilog("Switching to instant finality at block ${b}", ("b", blk->block_num));
                // switching from dpos to hotstuff, all nodes will switch at same block height
                // block header extension is set in finalize_block to value set by host function set_finalizers
                _chain->set_hs_irreversible_block_num(blk->block_num); // can be any value <= dpos lib

tests/TestHarness/Cluster.py

@@ -167,7 +167,8 @@ def setAlternateVersionLabels(self, file):
     # pylint: disable=too-many-statements
     def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=21, topo="mesh", delay=2, onlyBios=False, dontBootstrap=False,
                totalProducers=None, sharedProducers=0, extraNodeosArgs="", specificExtraNodeosArgs=None, specificNodeosInstances=None, onlySetProds=False,
-               pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True, nodeosLogPath=Path(Utils.TestLogRoot) / Path(f'{Path(sys.argv[0]).stem}{os.getpid()}'), genesisPath=None,
+               pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True, activateIF=False,
+               nodeosLogPath=Path(Utils.TestLogRoot) / Path(f'{Path(sys.argv[0]).stem}{os.getpid()}'), genesisPath=None,
                maximumP2pPerHost=0, maximumClients=25, prodsEnableTraceApi=True):
         """Launch cluster.
         pnodes: producer nodes count
@@ -519,7 +520,7 @@ def connectGroup(group, producerNodes, bridgeNodes) :
             return True
 
         Utils.Print("Bootstrap cluster.")
-        if not self.bootstrap(self.biosNode, self.startedNodesCount, prodCount + sharedProducers, totalProducers, pfSetupPolicy, onlyBios, onlySetProds, loadSystemContract):
+        if not self.bootstrap(launcher, self.biosNode, self.startedNodesCount, prodCount + sharedProducers, totalProducers, pfSetupPolicy, onlyBios, onlySetProds, loadSystemContract, activateIF):
             Utils.Print("ERROR: Bootstrap failed.")
             return False
 
@@ -991,7 +992,42 @@ def parseClusterKeys(totalNodes):
         Utils.Print(f'Found {len(producerKeys)} producer keys')
         return producerKeys
 
-    def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPolicy, onlyBios=False, onlySetProds=False, loadSystemContract=True):
+    def activateInstantFinality(self, launcher):
+        # call setfinalizer
+        numFins = len(launcher.network.nodes.values())
+        setFinStr =  f'{{"finalizer_policy": {{'
+        setFinStr += f'  "threshold": {int(numFins * 2 / 3 + 1)}, '
+        setFinStr += f'  "finalizers": ['
+        finNum = 1
+        for n in launcher.network.nodes.values():
+            if n.keys[0].blspubkey is None:
+                continue
+            if len(n.producers) == 0:
+                continue
+            setFinStr += f'    {{"description": "finalizer #{finNum}", '
+            setFinStr += f'     "weight":1, '
+            setFinStr += f'     "public_key": "{n.keys[0].blspubkey}", '
+            setFinStr += f'     "pop": "{n.keys[0].blspop}"'
+            setFinStr += f'    }}'
+            if finNum != numFins:
+                setFinStr += f', '
+            finNum = finNum + 1
+        setFinStr +=  f'  ]'
+        setFinStr +=  f'}}}}'
+        if Utils.Debug: Utils.Print("setfinalizers: %s" % (setFinStr))
+        Utils.Print("Setting finalizers")
+        opts = "--permission eosio@active"
+        trans = self.biosNode.pushMessage("eosio", "setfinalizer", setFinStr, opts)
+        if trans is None or not trans[0]:
+            Utils.Print("ERROR: Failed to set finalizers")
+            return None
+        Node.validateTransaction(trans[1])
+        transId = Node.getTransId(trans[1])
+        if not self.biosNode.waitForTransFinalization(transId, timeout=21*12*3):
+            Utils.Print("ERROR: Failed to validate transaction %s got rolled into a LIB block on server port %d." % (transId, biosNode.port))
+            return None
+
+    def bootstrap(self, launcher,  biosNode, totalNodes, prodCount, totalProducers, pfSetupPolicy, onlyBios=False, onlySetProds=False, loadSystemContract=True, activateIF=False):
         """Create 'prodCount' init accounts and deposits 10000000000 SYS in each. If prodCount is -1 will initialize all possible producers.
         Ensure nodes are inter-connected prior to this call. One way to validate this will be to check if every node has block 1."""
 
@@ -1027,12 +1063,11 @@ def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPoli
             Utils.Print("ERROR: Failed to import %s account keys into ignition wallet." % (eosioName))
             return None
 
-        contract="eosio.bios"
-        contractDir= str(self.libTestingContractsPath / contract)
-        if PFSetupPolicy.hasPreactivateFeature(pfSetupPolicy):
-            contractDir=str(self.libTestingContractsPath / "old_versions" / "v1.7.0-develop-preactivate_feature" / contract)
-        else:
-            contractDir=str(self.libTestingContractsPath / "old_versions" / "v1.6.0-rc3" / contract)
+        if not PFSetupPolicy.hasPreactivateFeature(pfSetupPolicy):
+            return True
+
+        contract="eosio.boot"
+        contractDir= str(self.unittestsContractsPath / contract)
         wasmFile="%s.wasm" % (contract)
         abiFile="%s.abi" % (contract)
         Utils.Print("Publish %s contract" % (contract))
@@ -1043,9 +1078,21 @@ def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPoli
 
         if pfSetupPolicy == PFSetupPolicy.FULL:
             biosNode.preactivateAllBuiltinProtocolFeature()
-
         Node.validateTransaction(trans)
 
+        contract="eosio.bios"
+        contractDir= str(self.libTestingContractsPath / contract)
+        wasmFile="%s.wasm" % (contract)
+        abiFile="%s.abi" % (contract)
+        Utils.Print("Publish %s contract" % (contract))
+        trans=biosNode.publishContract(eosioAccount, contractDir, wasmFile, abiFile, waitForTransBlock=True)
+        if trans is None:
+            Utils.Print("ERROR: Failed to publish contract %s." % (contract))
+            return None
+
+        if activateIF:
+            self.activateInstantFinality(launcher)
+
         Utils.Print("Creating accounts: %s " % ", ".join(producerKeys.keys()))
         producerKeys.pop(eosioName)
         accounts=[]
@@ -1092,7 +1139,7 @@ def bootstrap(self, biosNode, totalNodes, prodCount, totalProducers, pfSetupPoli
                     if counts[keys["node"]] >= prodCount:
                         Utils.Print(f'Count for this node exceeded: {counts[keys["node"]]}')
                         continue
-                    prodStanzas.append({ 'producer_name': keys['name'], 'block_signing_key': keys['public'] })
+                    prodStanzas.append({ 'producer_name': keys['name'], 'authority': ["block_signing_authority_v0", { 'threshold': 1, 'keys': [{ 'key': keys['public'], 'weight': 1 }]}]})
                     prodNames.append(keys["name"])
                     counts[keys["node"]] += 1
                 setProdsStr += json.dumps(prodStanzas)

tests/TestHarness/TestHelper.py

@@ -97,6 +97,9 @@ def createArgumentParser(includeArgs, applicationSpecificArgs=AppArgs(), suppres
         if "--dont-launch" in includeArgs:
             thGrp.add_argument("--dont-launch", help=argparse.SUPPRESS if suppressHelp else "Don't launch own node. Assume node is already running.",
                                      action='store_true')
+        if "--activate-if" in includeArgs:
+            thGrp.add_argument("--activate-if", help=argparse.SUPPRESS if suppressHelp else "Activate instant finality during bios boot.",
+                                     action='store_true')
         if "--keep-logs" in includeArgs:
             thGrp.add_argument("--keep-logs", help=argparse.SUPPRESS if suppressHelp else "Don't delete <test_name><pid>/node_* folders, or other test specific log directories, upon test completion",
                                      action='store_true')

tests/TestHarness/accounts.py

@@ -49,6 +49,9 @@ def __init__(self, name):
         self.ownerPublicKey=None
         self.activePrivateKey=None
         self.activePublicKey=None
+        self.blsFinalizerPrivateKey=None
+        self.blsFinalizerPublicKey=None
+        self.blsFinalizerPOP=None
 
 
     def __str__(self):
@@ -84,12 +87,28 @@ def createAccountKeys(count: int) -> List[Account]:
             activePrivate=m.group(1)
             activePublic=m.group(2)
 
+            # Private key: PVT_BLS_kRhJJ2MsM+/CddO...
+            # Public key: PUB_BLS_lbUE8922wUfX0Iy5...
+            # Proof of Possession: SIG_BLS_3jwkVUUYahHgsnmnEA...
+            rslts = Utils.processLeapUtilCmd("bls create key --to-console", "create key to console", silentErrors=False)
+            pattern = r'(\w+[^:]*): ([^\n]+)'
+            matched = re.findall(pattern, rslts)
+            results = {}
+            for k, v in matched:
+                results[k.strip()] = v.strip()
+            assert "PVT_BLS_" in results["Private key"]
+            assert "PUB_BLS_" in results["Public key"]
+            assert "SIG_BLS_" in results["Proof of Possession"]
+
             name=''.join(random.choice(string.ascii_lowercase) for _ in range(12))
             account=Account(name)
             account.ownerPrivateKey=ownerPrivate
             account.ownerPublicKey=ownerPublic
             account.activePrivateKey=activePrivate
             account.activePublicKey=activePublic
+            account.blsFinalizerPrivateKey=results["Private key"]
+            account.blsFinalizerPublicKey=results["Public key"]
+            account.blsFinalizerPOP=results["Proof of Possession"]
             accounts.append(account)
             if Utils.Debug: Utils.Print("name: %s, key(owner): ['%s', '%s], key(active): ['%s', '%s']" % (name, ownerPublic, ownerPrivate, activePublic, activePrivate))
 

tests/TestHarness/launcher.py

@@ -30,6 +30,9 @@ def default(self, o):
 class KeyStrings(object):
     pubkey: str
     privkey: str
+    blspubkey: str = None
+    blsprivkey: str = None
+    blspop: str = None
 
 @dataclass
 class nodeDefinition:
@@ -291,7 +294,7 @@ def bind_nodes(self):
                                             '5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3'))
                 node.producers.append('eosio')
             else:
-                node.keys.append(KeyStrings(account.ownerPublicKey, account.ownerPrivateKey))
+                node.keys.append(KeyStrings(account.ownerPublicKey, account.ownerPrivateKey, account.blsFinalizerPublicKey, account.blsFinalizerPrivateKey, account.blsFinalizerPOP))
                 if i < non_bios:
                     count = per_node
                     if extra:
@@ -479,7 +482,10 @@ def make_custom(self):
                 node = topo['nodes'][nodeName]
                 self.network.nodes[nodeName].dont_start = node['dont_start']
                 for keyObj in node['keys']:
-                    self.network.nodes[nodeName].keys.append(KeyStrings(keyObj['pubkey'], keyObj['privkey']))
+                    if 'blspubkey' in keyObj:
+                        self.network.nodes[nodeName].keys.append(KeyStrings(keyObj['pubkey'], keyObj['privkey'], keyObj['blspubkey'], keyObj['blsprivkey'], keyObj['blspop']))
+                    else:
+                        self.network.nodes[nodeName].keys.append(KeyStrings(keyObj['pubkey'], keyObj['privkey']))
                 for peer in node['peers']:
                     self.network.nodes[nodeName].peers.append(peer)
                 for producer in node['producers']:
@@ -508,6 +514,8 @@ def construct_command_line(self, instance: nodeDefinition):
             a(a(eosdcmd, '--plugin'), 'eosio::producer_plugin')
             producer_keys = list(sum([('--signature-provider', f'{key.pubkey}=KEY:{key.privkey}') for key in instance.keys], ()))
             eosdcmd.extend(producer_keys)
+            finalizer_keys = list(sum([('--signature-provider', f'{key.blspubkey}=KEY:{key.blsprivkey}') for key in instance.keys], ()))
+            eosdcmd.extend(finalizer_keys)
             producer_names = list(sum([('--producer-name', p) for p in instance.producers], ()))
             eosdcmd.extend(producer_names)
         else:

tests/TestHarness/logging-template.json

@@ -130,13 +130,21 @@
         "stderr"
       ]
     },{
-    "name": "state_history",
-    "level": "info",
-    "enabled": true,
-    "additivity": false,
-    "appenders": [
-      "stderr"
-      ]
+       "name": "state_history",
+       "level": "info",
+       "enabled": true,
+       "additivity": false,
+       "appenders": [
+         "stderr"
+        ]
+    },{
+       "name": "hotstuff",
+       "level": "all",
+       "enabled": true,
+       "additivity": false,
+       "appenders": [
+         "stderr"
+       ]
     },{
       "name": "transaction",
       "level": "info",

tests/distributed-transactions-test.py

@@ -23,7 +23,7 @@
 
 appArgs = AppArgs()
 extraArgs = appArgs.add_bool(flag="--speculative", help="Run nodes in read-mode=speculative")
-args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed", "--speculative"
+args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed", "--speculative", "--activate-if"
                            ,"--dump-error-details","-v","--leave-running","--keep-logs","--unshared"}, applicationSpecificArgs=appArgs)
 
 pnodes=args.p
@@ -36,12 +36,13 @@
 seed=args.seed
 dumpErrorDetails=args.dump_error_details
 speculative=args.speculative
+activateIF=args.activate_if
 
 Utils.Debug=debug
 testSuccessful=False
 
 random.seed(seed) # Use a fixed seed for repeatability.
-cluster=Cluster(unshared=args.unshared, keepRunning=True if nodesFile is not None else args.leave_running, keepLogs=args.keep_logs)
+cluster=Cluster(unshared=args.unshared, keepRunning=True if nodesFile is not None else args.leave_running, keepLogs=args.keep_logs, loggingLevel="all")
 walletMgr=WalletMgr(True)
 
 try:
@@ -67,7 +68,7 @@
         if speculative:
            extraNodeosArgs = " --read-mode speculative "
 
-        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, extraNodeosArgs=extraNodeosArgs) is False:
+        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, extraNodeosArgs=extraNodeosArgs, activateIF=activateIF) is False:
             errorExit("Failed to stand up eos cluster.")
 
         Print ("Wait for Cluster stabilization")

tests/nodeos_chainbase_allocation_test.py

@@ -61,7 +61,7 @@
     nonProdNode.createAccount(newProducerAcc, cluster.eosioAccount, waitForTransBlock=True)
 
     setProdsStr = '{"schedule": ['
-    setProdsStr += '{"producer_name":' + newProducerAcc.name + ',"block_signing_key":' + newProducerAcc.activePublicKey + '}'
+    setProdsStr += '{"producer_name":' + newProducerAcc.name + ',"authority": ["block_signing_authority_v0", {"threshold":1, "keys":[{"key":' + newProducerAcc.activePublicKey + ', "weight":1}]}]}'
     setProdsStr += ']}'
     cmd="push action -j eosio setprods '{}' -p eosio".format(setProdsStr)
     trans = producerNode.processCleosCmd(cmd, cmd, silentErrors=False)

tests/nodeos_extra_packed_data_test.py

@@ -69,7 +69,6 @@
         if cluster.launch(totalNodes=totalNodes, 
                           pnodes=pnodes,
                           dontBootstrap=dontBootstrap,
-                          pfSetupPolicy=PFSetupPolicy.PREACTIVATE_FEATURE_ONLY,
                           specificExtraNodeosArgs=specificExtraNodeosArgs,
                           associatedNodeLabels=associatedNodeLabels) is False:
             cmdError("launcher")

tests/nodeos_producer_watermark_test.py

@@ -40,7 +40,7 @@ def setProds(sharedProdKey):
         key = cluster.defProducerAccounts[name].activePublicKey
         if name == "shrproducera":
             key = sharedProdKey
-        setProdsStr += ' { "producer_name": "%s", "block_signing_key": "%s" }' % (name, key)
+        setProdsStr += '{"producer_name":' + name + ',"authority": ["block_signing_authority_v0", {"threshold":1, "keys":[{"key":' + key + ', "weight":1}]}]}'
 
     setProdsStr += ' ] }'
     Utils.Print("setprods: %s" % (setProdsStr))

tests/nodeos_run_test.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 
 from TestHarness import Account, Cluster, Node, ReturnType, TestHelper, Utils, WalletMgr, CORE_SYMBOL, createAccountKeys
-from pathlib import Path
 
 import decimal
 import re
@@ -22,7 +21,7 @@
 
 args = TestHelper.parse_args({"--host","--port","--prod-count","--defproducera_prvt_key","--defproducerb_prvt_key"
                               ,"--dump-error-details","--dont-launch","--keep-logs","-v","--leave-running","--only-bios"
-                              ,"--sanity-test","--wallet-port", "--error-log-path", "--unshared"})
+                              ,"--activate-if","--sanity-test","--wallet-port", "--error-log-path", "--unshared"})
 server=args.host
 port=args.port
 debug=args.v
@@ -34,6 +33,7 @@
 onlyBios=args.only_bios
 sanityTest=args.sanity_test
 walletPort=args.wallet_port
+activateIF=args.activate_if
 
 Utils.Debug=debug
 localTest=True if server == TestHelper.LOCAL_HOST else False
@@ -63,7 +63,7 @@
         traceNodeosArgs=" --http-max-response-time-ms 990000 --trace-rpc-abi eosio.token=" + abs_path
         extraNodeosArgs=traceNodeosArgs + " --plugin eosio::prometheus_plugin --database-map-mode mapped_private "
         specificNodeosInstances={0: "bin/nodeos"}
-        if cluster.launch(totalNodes=2, prodCount=prodCount, onlyBios=onlyBios, dontBootstrap=dontBootstrap, extraNodeosArgs=extraNodeosArgs, specificNodeosInstances=specificNodeosInstances) is False:
+        if cluster.launch(totalNodes=2, prodCount=prodCount, activateIF=activateIF, onlyBios=onlyBios, dontBootstrap=dontBootstrap, extraNodeosArgs=extraNodeosArgs, specificNodeosInstances=specificNodeosInstances) is False:
             cmdError("launcher")
             errorExit("Failed to stand up eos cluster.")
     else: