[PH] Feature performance harness stage 2 development

tests/TestHarness/Cluster.py

@@ -94,7 +94,7 @@ def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888
         defproduceraPrvtKey: Defproducera account private key
         defproducerbPrvtKey: Defproducerb account private key
         """
-        self.accounts={}
+        self.accounts=[]
         self.nodes=[]
         self.unstartedNodes=[]
         self.localCluster=localCluster
@@ -257,11 +257,18 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me
         cmdArr=cmd.split()
         if self.staging:
             cmdArr.append("--nogen")
-
-        nodeosArgs="--max-transaction-time -1 --abi-serializer-max-time-ms 990000 --p2p-max-nodes-per-host %d --max-clients %d" % (maximumP2pPerHost, maximumClients)
+        nodeosArgs=""
+        if "--max-transaction-time" not in extraNodeosArgs:
+            nodeosArgs += " --max-transaction-time -1"
+        if "--abi-serializer-max-time-ms" not in extraNodeosArgs:
+            nodeosArgs += " --abi-serializer-max-time-ms 990000"
+        if "--p2p-max-nodes-per-host" not in extraNodeosArgs:
+            nodeosArgs += f" --p2p-max-nodes-per-host {maximumP2pPerHost}"
+        if "--max-clients" not in extraNodeosArgs:
+            nodeosArgs += f" --max-clients {maximumClients}"
         if not self.walletd:
             nodeosArgs += " --plugin eosio::wallet_api_plugin"
-        if Utils.Debug:
+        if Utils.Debug and "--contracts-console" not in extraNodeosArgs:
             nodeosArgs += " --contracts-console"
         if PFSetupPolicy.hasPreactivateFeature(pfSetupPolicy):
             nodeosArgs += " --plugin eosio::producer_api_plugin"
@@ -705,11 +712,16 @@ def createAccountKeys(count):
 
     # create account keys and import into wallet. Wallet initialization will be user responsibility
     # also imports defproducera and defproducerb accounts
-    def populateWallet(self, accountsCount, wallet):
+    def populateWallet(self, accountsCount, wallet, accountNames: list=None, createProducerAccounts: bool=True):
+        if accountsCount == 0 and (accountNames is None or len(accountNames) == 0):
+            return True
         if self.walletMgr is None:
             Utils.Print("ERROR: WalletMgr hasn't been initialized.")
             return False
 
+        if accountNames is not None:
+            assert(len(accountNames) <= accountsCount)
+
         accounts=None
         if accountsCount > 0:
             Utils.Print ("Create account keys.")
@@ -718,23 +730,28 @@ def populateWallet(self, accountsCount, wallet):
                 Utils.Print("Account keys creation failed.")
                 return False
 
-        Utils.Print("Importing keys for account %s into wallet %s." % (self.defproduceraAccount.name, wallet.name))
-        if not self.walletMgr.importKey(self.defproduceraAccount, wallet):
-            Utils.Print("ERROR: Failed to import key for account %s" % (self.defproduceraAccount.name))
-            return False
+        if createProducerAccounts:
+            Utils.Print("Importing keys for account %s into wallet %s." % (self.defproduceraAccount.name, wallet.name))
+            if not self.walletMgr.importKey(self.defproduceraAccount, wallet):
+                Utils.Print("ERROR: Failed to import key for account %s" % (self.defproduceraAccount.name))
+                return False
 
-        Utils.Print("Importing keys for account %s into wallet %s." % (self.defproducerbAccount.name, wallet.name))
-        if not self.walletMgr.importKey(self.defproducerbAccount, wallet):
-            Utils.Print("ERROR: Failed to import key for account %s" % (self.defproducerbAccount.name))
-            return False
+            Utils.Print("Importing keys for account %s into wallet %s." % (self.defproducerbAccount.name, wallet.name))
+            if not self.walletMgr.importKey(self.defproducerbAccount, wallet):
+                Utils.Print("ERROR: Failed to import key for account %s" % (self.defproducerbAccount.name))
+                return False
+
+        if accountNames is not None:
+            for idx, name in enumerate(accountNames):
+                accounts[idx].name =  name
 
         for account in accounts:
             Utils.Print("Importing keys for account %s into wallet %s." % (account.name, wallet.name))
             if not self.walletMgr.importKey(account, wallet):
                 Utils.Print("ERROR: Failed to import key for account %s" % (account.name))
                 return False
+            self.accounts.append(account)
 
-        self.accounts=accounts
         return True
 
     def getNodeP2pPort(self, nodeId: int):
@@ -1587,22 +1604,22 @@ def cleanup(self):
         for f in self.filesToCleanup:
             os.remove(f)
 
-
-    # Create accounts and validates that the last transaction is received on root node
+    # Create accounts, if account does not already exist, and validates that the last transaction is received on root node
     def createAccounts(self, creator, waitForTransBlock=True, stakedDeposit=1000, validationNodeIndex=0):
         if self.accounts is None:
             return True
-
         transId=None
         for account in self.accounts:
-            if Utils.Debug: Utils.Print("Create account %s." % (account.name))
-            if Utils.Debug: Utils.Print("Validation node %s" % validationNodeIndex)
-            trans=self.createAccountAndVerify(account, creator, stakedDeposit, validationNodeIndex=validationNodeIndex)
-            if trans is None:
-                Utils.Print("ERROR: Failed to create account %s." % (account.name))
-                return False
-            if Utils.Debug: Utils.Print("Account %s created." % (account.name))
-            transId=Node.getTransId(trans)
+            ret = self.biosNode.getEosAccount(account.name)
+            if ret is None:
+                if Utils.Debug: Utils.Print("Create account %s." % (account.name))
+                if Utils.Debug: Utils.Print("Validation node %s" % validationNodeIndex)
+                trans=self.createAccountAndVerify(account, creator, stakedDeposit, validationNodeIndex=validationNodeIndex)
+                if trans is None:
+                    Utils.Print("ERROR: Failed to create account %s." % (account.name))
+                    return False
+                if Utils.Debug: Utils.Print("Account %s created." % (account.name))
+                transId=Node.getTransId(trans)
 
         if waitForTransBlock and transId is not None:
             node=self.nodes[validationNodeIndex]
@@ -1760,7 +1777,7 @@ def stripValues(lowestMaxes,greaterThan):
 
     def launchTrxGenerators(self, contractOwnerAcctName: str, acctNamesList: list, acctPrivKeysList: list,
                             nodeId: int=0, tpsPerGenerator: int=10, numGenerators: int=1, durationSec: int=60,
-                            waitToComplete:bool=False):
+                            waitToComplete:bool=False, abiFile=None, actionsData=None, actionsAuths=None):
         Utils.Print("Configure txn generators")
         node=self.getNode(nodeId)
         p2pListenPort = self.getNodeP2pPort(nodeId)
@@ -1775,9 +1792,10 @@ def launchTrxGenerators(self, contractOwnerAcctName: str, acctNamesList: list, a
 
         tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator)
         self.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id,
-                                                    handlerAcct=contractOwnerAcctName, accts=','.join(map(str, acctNamesList)),
-                                                    privateKeys=','.join(map(str, acctPrivKeysList)), trxGenDurationSec=durationSec,
-                                                    logDir=Utils.DataDir, peerEndpoint=self.host, port=p2pListenPort, tpsTrxGensConfig=tpsTrxGensConfig)
+                                                    contractOwnerAccount=contractOwnerAcctName, accts=','.join(map(str, acctNamesList)),
+                                                    privateKeys=','.join(map(str, acctPrivKeysList)), trxGenDurationSec=durationSec, logDir=Utils.DataDir,
+                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths,
+                                                    peerEndpoint=self.host, port=p2pListenPort, tpsTrxGensConfig=tpsTrxGensConfig)
 
         Utils.Print("Launch txn generators and start generating/sending transactions")
         self.trxGenLauncher.launch(waitToComplete=waitToComplete)

tests/TestHarness/Node.py

@@ -518,18 +518,18 @@ def checkBlockForTransactions(self, transIds, blockNum):
         return transIds
 
     def waitForTransactionsInBlockRange(self, transIds, startBlock=2, maxFutureBlocks=0):
-        lastBlockProcessed = startBlock
-        overallFinalBlock = startBlock + maxFutureBlocks
+        nextBlockToProcess = startBlock
+        overallEndBlock = startBlock + maxFutureBlocks
         while len(transIds) > 0:
             currentLoopEndBlock = self.getHeadBlockNum()
-            if currentLoopEndBlock >  overallFinalBlock:
-                currentLoopEndBlock = overallFinalBlock
-            for blockNum in range(currentLoopEndBlock, lastBlockProcessed - 1, -1):
+            if currentLoopEndBlock >  overallEndBlock:
+                currentLoopEndBlock = overallEndBlock
+            for blockNum in range(nextBlockToProcess, currentLoopEndBlock + 1):
                 transIds = self.checkBlockForTransactions(transIds, blockNum)
                 if len(transIds) == 0:
                     return transIds
-            lastBlockProcessed = currentLoopEndBlock
-            if currentLoopEndBlock == overallFinalBlock:
+            nextBlockToProcess = currentLoopEndBlock + 1
+            if currentLoopEndBlock == overallEndBlock:
                 Utils.Print("ERROR: Transactions were missing upon expiration of waitOnblockTransactions")
                 break
             self.waitForHeadToAdvance()

tests/TestHarness/TestHelper.py

@@ -37,87 +37,100 @@ class TestHelper(object):
     @staticmethod
     # pylint: disable=too-many-branches
     # pylint: disable=too-many-statements
-    def parse_args(includeArgs, applicationSpecificArgs=AppArgs()):
+    def createArgumentParser(includeArgs, applicationSpecificArgs=AppArgs()) -> argparse.ArgumentParser:
         """Accepts set of arguments, builds argument parser and returns parse_args() output."""
         assert(includeArgs)
         assert(isinstance(includeArgs, set))
         assert(isinstance(applicationSpecificArgs, AppArgs))
 
-        parser = argparse.ArgumentParser(add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-        parser.add_argument('-?', action='help', default=argparse.SUPPRESS,
-                                 help=argparse._('show this help message and exit'))
+        thParser = argparse.ArgumentParser(add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        thGrpTitle = "Test Helper Arguments"
+        thGrpDescription="Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment."
+        thGrp = thParser.add_argument_group(title=thGrpTitle, description=thGrpDescription)
+        thGrp.add_argument('-?', '--help', action='help', default=argparse.SUPPRESS, help=argparse._('show this help message and exit'))
 
         if "-p" in includeArgs:
-            parser.add_argument("-p", type=int, help="producing nodes count", default=1)
+            thGrp.add_argument("-p", type=int, help="producing nodes count", default=1)
         if "-n" in includeArgs:
-            parser.add_argument("-n", type=int, help="total nodes", default=0)
+            thGrp.add_argument("-n", type=int, help="total nodes", default=0)
         if "-d" in includeArgs:
-            parser.add_argument("-d", type=int, help="delay between nodes startup", default=1)
+            thGrp.add_argument("-d", type=int, help="delay between nodes startup", default=1)
         if "--nodes-file" in includeArgs:
-            parser.add_argument("--nodes-file", type=str, help="File containing nodes info in JSON format.")
+            thGrp.add_argument("--nodes-file", type=str, help="File containing nodes info in JSON format.")
         if "-s" in includeArgs:
-            parser.add_argument("-s", type=str, help="topology", choices=["mesh"], default="mesh")
+            thGrp.add_argument("-s", type=str, help="topology", choices=["mesh"], default="mesh")
         if "-c" in includeArgs:
-            parser.add_argument("-c", type=str, help="chain strategy",
+            thGrp.add_argument("-c", type=str, help="chain strategy",
                     choices=[Utils.SyncResyncTag, Utils.SyncReplayTag, Utils.SyncNoneTag, Utils.SyncHardReplayTag],
                     default=Utils.SyncResyncTag)
         if "--kill-sig" in includeArgs:
-            parser.add_argument("--kill-sig", type=str, choices=[Utils.SigKillTag, Utils.SigTermTag], help="kill signal.",
+            thGrp.add_argument("--kill-sig", type=str, choices=[Utils.SigKillTag, Utils.SigTermTag], help="kill signal.",
                     default=Utils.SigKillTag)
         if "--kill-count" in includeArgs:
-            parser.add_argument("--kill-count", type=int, help="nodeos instances to kill", default=-1)
+            thGrp.add_argument("--kill-count", type=int, help="nodeos instances to kill", default=-1)
         if "--terminate-at-block" in includeArgs:
-            parser.add_argument("--terminate-at-block", type=int, help="block to terminate on when replaying", default=0)
+            thGrp.add_argument("--terminate-at-block", type=int, help="block to terminate on when replaying", default=0)
         if "--seed" in includeArgs:
-            parser.add_argument("--seed", type=int, help="random seed", default=1)
+            thGrp.add_argument("--seed", type=int, help="random seed", default=1)
 
         if "--host" in includeArgs:
-            parser.add_argument("-h", "--host", type=str, help="%s host name" % (Utils.EosServerName),
+            thGrp.add_argument("-h", "--host", type=str, help="%s host name" % (Utils.EosServerName),
                                      default=TestHelper.LOCAL_HOST)
         if "--port" in includeArgs:
-            parser.add_argument("--port", type=int, help="%s host port" % Utils.EosServerName,
+            thGrp.add_argument("--port", type=int, help="%s host port" % Utils.EosServerName,
                                      default=TestHelper.DEFAULT_PORT)
         if "--wallet-host" in includeArgs:
-            parser.add_argument("--wallet-host", type=str, help="%s host" % Utils.EosWalletName,
+            thGrp.add_argument("--wallet-host", type=str, help="%s host" % Utils.EosWalletName,
                                      default=TestHelper.LOCAL_HOST)
         if "--wallet-port" in includeArgs:
-            parser.add_argument("--wallet-port", type=int, help="%s port" % Utils.EosWalletName,
+            thGrp.add_argument("--wallet-port", type=int, help="%s port" % Utils.EosWalletName,
                                      default=TestHelper.DEFAULT_WALLET_PORT)
         if "--prod-count" in includeArgs:
-            parser.add_argument("-c", "--prod-count", type=int, help="Per node producer count", default=1)
+            thGrp.add_argument("-c", "--prod-count", type=int, help="Per node producer count", default=1)
         if "--defproducera_prvt_key" in includeArgs:
-            parser.add_argument("--defproducera_prvt_key", type=str, help="defproducera private key.")
+            thGrp.add_argument("--defproducera_prvt_key", type=str, help="defproducera private key.")
         if "--defproducerb_prvt_key" in includeArgs:
-            parser.add_argument("--defproducerb_prvt_key", type=str, help="defproducerb private key.")
+            thGrp.add_argument("--defproducerb_prvt_key", type=str, help="defproducerb private key.")
         if "--dump-error-details" in includeArgs:
-            parser.add_argument("--dump-error-details",
+            thGrp.add_argument("--dump-error-details",
                                      help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout",
                                      action='store_true')
         if "--dont-launch" in includeArgs:
-            parser.add_argument("--dont-launch", help="Don't launch own node. Assume node is already running.",
+            thGrp.add_argument("--dont-launch", help="Don't launch own node. Assume node is already running.",
                                      action='store_true')
         if "--keep-logs" in includeArgs:
-            parser.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders, or other test specific log directories, upon test completion",
+            thGrp.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders, or other test specific log directories, upon test completion",
                                      action='store_true')
         if "-v" in includeArgs:
-            parser.add_argument("-v", help="verbose logging", action='store_true')
+            thGrp.add_argument("-v", help="verbose logging", action='store_true')
         if "--leave-running" in includeArgs:
-            parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true')
+            thGrp.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true')
         if "--only-bios" in includeArgs:
-            parser.add_argument("--only-bios", help="Limit testing to bios node.", action='store_true')
+            thGrp.add_argument("--only-bios", help="Limit testing to bios node.", action='store_true')
         if "--clean-run" in includeArgs:
-            parser.add_argument("--clean-run", help="Kill all nodeos and keosd instances", action='store_true')
+            thGrp.add_argument("--clean-run", help="Kill all nodeos and keosd instances", action='store_true')
         if "--sanity-test" in includeArgs:
-            parser.add_argument("--sanity-test", help="Validates nodeos and keosd are in path and can be started up.", action='store_true')
+            thGrp.add_argument("--sanity-test", help="Validates nodeos and keosd are in path and can be started up.", action='store_true')
         if "--alternate-version-labels-file" in includeArgs:
-            parser.add_argument("--alternate-version-labels-file", type=str, help="Provide a file to define the labels that can be used in the test and the path to the version installation associated with that.")
+            thGrp.add_argument("--alternate-version-labels-file", type=str, help="Provide a file to define the labels that can be used in the test and the path to the version installation associated with that.")
+
+        if len(applicationSpecificArgs.args) > 0:
+            appArgsGrpTitle="Application Specific Arguments"
+            appArgsGrpdescription="Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment."
+            appArgsGrp = thParser.add_argument_group(title=appArgsGrpTitle, description=appArgsGrpdescription)
+            for arg in applicationSpecificArgs.args:
+                if arg.type is not None:
+                    appArgsGrp.add_argument(arg.flag, type=arg.type, help=arg.help, choices=arg.choices, default=arg.default)
+                else:
+                    appArgsGrp.add_argument(arg.flag, help=arg.help, action=arg.action)
 
-        for arg in applicationSpecificArgs.args:
-            if arg.type is not None:
-                parser.add_argument(arg.flag, type=arg.type, help=arg.help, choices=arg.choices, default=arg.default)
-            else:
-                parser.add_argument(arg.flag, help=arg.help, action=arg.action)
+        return thParser
 
+    @staticmethod
+    # pylint: disable=too-many-branches
+    # pylint: disable=too-many-statements
+    def parse_args(includeArgs, applicationSpecificArgs=AppArgs()):
+        parser = TestHelper.createArgumentParser(includeArgs=includeArgs, applicationSpecificArgs=applicationSpecificArgs)
         args = parser.parse_args()
         return args