From 6a49cbc94d24cc658e9fb24b97aaeb614fb1e5f3 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 30 Aug 2023 13:16:01 -0500 Subject: [PATCH 1/4] Make transaction generator a configurable option to PerformanceHarness. --- tests/PerformanceHarness/performance_test.py | 8 ++++++-- tests/PerformanceHarness/performance_test_basic.py | 5 +++-- tests/PerformanceHarnessScenarioRunner.py | 6 ++++-- tests/TestHarness/Cluster.py | 5 +++-- tests/TestHarness/launch_transaction_generators.py | 11 +++++++---- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/PerformanceHarness/performance_test.py b/tests/PerformanceHarness/performance_test.py index 797654fc3a..4332450304 100755 --- a/tests/PerformanceHarness/performance_test.py +++ b/tests/PerformanceHarness/performance_test.py @@ -49,6 +49,8 @@ class PtConfig: userTrxDataFile: Path=None endpointMode: str="p2p" opModeCmd: str="" + trxGenerator: Path=Path(".") + def __post_init__(self): self.opModeDesc = "Block Producer Operational Mode" if self.opModeCmd == "testBpOpMode" else "API Node Operational Mode" if self.opModeCmd == "testApiOpMode" else "Undefined Operational Mode" @@ -112,7 +114,8 @@ def performPtbBinarySearch(self, clusterConfig: PerformanceTestBasic.ClusterConf scenarioResult = PerformanceTest.PerfTestSearchIndivResult(success=False, searchTarget=binSearchTarget, searchFloor=floor, searchCeiling=ceiling) ptbConfig = PerformanceTestBasic.PtbConfig(targetTps=binSearchTarget, testTrxGenDurationSec=self.ptConfig.testDurationSec, tpsLimitPerGenerator=self.ptConfig.tpsLimitPerGenerator, numAddlBlocksToPrune=self.ptConfig.numAddlBlocksToPrune, logDirRoot=logDirRoot, delReport=delReport, - quiet=quiet, userTrxDataFile=self.ptConfig.userTrxDataFile, endpointMode=self.ptConfig.endpointMode) + quiet=quiet, userTrxDataFile=self.ptConfig.userTrxDataFile, endpointMode=self.ptConfig.endpointMode, + trxGenerator=self.ptConfig.trxGenerator) myTest = PerformanceTestBasic(testHelperConfig=self.testHelperConfig, clusterConfig=clusterConfig, ptbConfig=ptbConfig, testNamePath=os.path.basename(sys.argv[0]).rsplit('.',maxsplit=1)[0]) myTest.runTest() @@ -154,7 +157,8 @@ def performPtbReverseLinearSearch(self, tpsInitial: int) -> TpsTestResult.PerfTe scenarioResult = PerformanceTest.PerfTestSearchIndivResult(success=False, searchTarget=searchTarget, searchFloor=absFloor, searchCeiling=absCeiling) ptbConfig = PerformanceTestBasic.PtbConfig(targetTps=searchTarget, testTrxGenDurationSec=self.ptConfig.testDurationSec, tpsLimitPerGenerator=self.ptConfig.tpsLimitPerGenerator, numAddlBlocksToPrune=self.ptConfig.numAddlBlocksToPrune, logDirRoot=self.loggingConfig.ptbLogsDirPath, delReport=self.ptConfig.delReport, - quiet=self.ptConfig.quiet, delPerfLogs=self.ptConfig.delPerfLogs, userTrxDataFile=self.ptConfig.userTrxDataFile, endpointMode=self.ptConfig.endpointMode) + quiet=self.ptConfig.quiet, delPerfLogs=self.ptConfig.delPerfLogs, userTrxDataFile=self.ptConfig.userTrxDataFile, endpointMode=self.ptConfig.endpointMode, + trxGenerator=self.ptConfig.trxGenerator) myTest = PerformanceTestBasic(testHelperConfig=self.testHelperConfig, clusterConfig=self.clusterConfig, ptbConfig=ptbConfig, testNamePath=os.path.basename(sys.argv[0]).rsplit('.',maxsplit=1)[0]) myTest.runTest() diff --git a/tests/PerformanceHarness/performance_test_basic.py b/tests/PerformanceHarness/performance_test_basic.py index 5189dc2530..d2d3b72fa0 100755 --- a/tests/PerformanceHarness/performance_test_basic.py +++ b/tests/PerformanceHarness/performance_test_basic.py @@ -173,7 +173,7 @@ class PtbConfig: userTrxDataFile: Path=None endpointMode: str="p2p" apiEndpoint: str=None - + trxGenerator: Path=Path(".") def __post_init__(self): self.expectedTransactionsSent = self.testTrxGenDurationSec * self.targetTps @@ -454,7 +454,7 @@ def configureConnections(): self.data.startBlock = self.waitForEmptyBlocks(self.validationNode, self.emptyBlockGoal) tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator, connectionPairList=self.connectionPairList) - self.cluster.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id, contractOwnerAccount=self.clusterConfig.specifiedContract.account.name, + self.cluster.trxGenLauncher = TransactionGeneratorsLauncher(trxGenerator=self.ptbConfig.trxGenerator, chainId=chainId, lastIrreversibleBlockId=lib_id, contractOwnerAccount=self.clusterConfig.specifiedContract.account.name, accts=','.join(map(str, self.accountNames)), privateKeys=','.join(map(str, self.accountPrivKeys)), trxGenDurationSec=self.ptbConfig.testTrxGenDurationSec, logDir=self.trxGenLogDirPath, abiFile=abiFile, actionsData=actionsDataJson, actionsAuths=actionsAuthsJson, @@ -722,6 +722,7 @@ def _createBaseArgumentParser(defEndpointApiDef: str, defProdNodeCnt: int, defVa block log file is removed after startup.", default=None) ptbBaseParserGroup.add_argument("--http-threads", type=int, help=argparse.SUPPRESS if suppressHelp else "Number of worker threads in http thread pool", default=2) ptbBaseParserGroup.add_argument("--chain-state-db-size-mb", type=int, help=argparse.SUPPRESS if suppressHelp else "Maximum size (in MiB) of the chain state database", default=25600) + ptbBaseParserGroup.add_argument("--trx-generator", type=str, help=argparse.SUPPRESS if suppressHelp else "Transaction Generator executable", default="./tests/trx_generator/trx_generator") return ptbBaseParser diff --git a/tests/PerformanceHarnessScenarioRunner.py b/tests/PerformanceHarnessScenarioRunner.py index ac0a3d54d3..564ca0cf19 100755 --- a/tests/PerformanceHarnessScenarioRunner.py +++ b/tests/PerformanceHarnessScenarioRunner.py @@ -63,7 +63,8 @@ def main(): delPerfLogs=args.del_perf_logs, printMissingTransactions=args.print_missing_transactions, userTrxDataFile=Path(args.user_trx_data_file) if args.user_trx_data_file is not None else None, - endpointMode=args.endpoint_mode) + endpointMode=args.endpoint_mode, + trxGenerator=args.trx_generator) Utils.Print(f"testNamePath: {PurePath(PurePath(__file__).name).stem}") myTest = performance_test_basic.PerformanceTestBasic(testHelperConfig=testHelperConfig, clusterConfig=testClusterConfig, ptbConfig=ptbConfig, testNamePath=f"{PurePath(PurePath(__file__).name).stem}") elif args.scenario_type_sub_cmd == "findMax": @@ -85,7 +86,8 @@ def main(): calcNetThreads=args.calc_net_threads, userTrxDataFile=Path(args.user_trx_data_file) if args.user_trx_data_file is not None else None, endpointMode=args.endpoint_mode, - opModeCmd=args.op_mode_sub_cmd) + opModeCmd=args.op_mode_sub_cmd, + trxGenerator=args.trx_generator) myTest = performance_test.PerformanceTest(testHelperConfig=testHelperConfig, clusterConfig=testClusterConfig, ptConfig=ptConfig) else: diff --git a/tests/TestHarness/Cluster.py b/tests/TestHarness/Cluster.py index c028b7a632..6d42864d0b 100644 --- a/tests/TestHarness/Cluster.py +++ b/tests/TestHarness/Cluster.py @@ -1554,7 +1554,8 @@ def stripValues(lowestMaxes,greaterThan): def launchTrxGenerators(self, contractOwnerAcctName: str, acctNamesList: list, acctPrivKeysList: list, nodeId: int=0, tpsPerGenerator: int=10, numGenerators: int=1, durationSec: int=60, - waitToComplete:bool=False, abiFile=None, actionsData=None, actionsAuths=None): + waitToComplete:bool=False, abiFile=None, actionsData=None, actionsAuths=None, + trxGenerator=Path("./tests/trx_generator/trx_generator")): Utils.Print("Configure txn generators") node=self.getNode(nodeId) info = node.getInfo() @@ -1567,7 +1568,7 @@ def launchTrxGenerators(self, contractOwnerAcctName: str, acctNamesList: list, a self.preExistingFirstTrxFiles = glob.glob(f"{Utils.DataDir}/first_trx_*.txt") connectionPairList = [f"{self.host}:{self.getNodeP2pPort(nodeId)}"] tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator, connectionPairList=connectionPairList) - self.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id, + self.trxGenLauncher = TransactionGeneratorsLauncher(trxGenerator=trxGenerator, chainId=chainId, lastIrreversibleBlockId=lib_id, contractOwnerAccount=contractOwnerAcctName, accts=','.join(map(str, acctNamesList)), privateKeys=','.join(map(str, acctPrivKeysList)), trxGenDurationSec=durationSec, logDir=Utils.DataDir, abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths, tpsTrxGensConfig=tpsTrxGensConfig, diff --git a/tests/TestHarness/launch_transaction_generators.py b/tests/TestHarness/launch_transaction_generators.py index 353e361333..22951108dc 100644 --- a/tests/TestHarness/launch_transaction_generators.py +++ b/tests/TestHarness/launch_transaction_generators.py @@ -37,8 +37,9 @@ def __init__(self, targetTps: int, tpsLimitPerGenerator: int, connectionPairList class TransactionGeneratorsLauncher: - def __init__(self, chainId: int, lastIrreversibleBlockId: int, contractOwnerAccount: str, accts: str, privateKeys: str, trxGenDurationSec: int, logDir: str, + def __init__(self, trxGenerator: Path, chainId: int, lastIrreversibleBlockId: int, contractOwnerAccount: str, accts: str, privateKeys: str, trxGenDurationSec: int, logDir: str, abiFile: Path, actionsData, actionsAuths, tpsTrxGensConfig: TpsTrxGensConfig, endpointMode: str, apiEndpoint: str=None): + self.trxGenerator = trxGenerator self.chainId = chainId self.lastIrreversibleBlockId = lastIrreversibleBlockId self.contractOwnerAccount = contractOwnerAccount @@ -59,7 +60,8 @@ def launch(self, waitToComplete=True): for id, targetTps in enumerate(self.tpsTrxGensConfig.targetTpsPerGenList): connectionPair = self.tpsTrxGensConfig.connectionPairList[connectionPairIter].rsplit(":") popenStringList = [ - './tests/trx_generator/trx_generator', + # './tests/trx_generator/trx_generator', + f'{self.trxGenerator}', '--generator-id', f'{id}', '--chain-id', f'{self.chainId}', '--last-irreversible-block-id', f'{self.lastIrreversibleBlockId}', @@ -80,7 +82,7 @@ def launch(self, waitToComplete=True): popenStringList.extend(['--api-endpoint', f'{self.apiEndpoint}']) if Utils.Debug: - Print(f"Running trx_generator: {' '.join(popenStringList)}") + Print(f"Running transaction generator {self.trxGenerator} : {' '.join(popenStringList)}") self.subprocess_ret_codes.append(subprocess.Popen(popenStringList)) connectionPairIter = (connectionPairIter + 1) % len(self.tpsTrxGensConfig.connectionPairList) exitCodes=None @@ -97,6 +99,7 @@ def killAll(self): def parseArgs(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument('-?', '--help', action='help', default=argparse.SUPPRESS, help=argparse._('show this help message and exit')) + parser.add_argument("trx_generator", type=str, help="The path to the transaction generator binary to excecute") parser.add_argument("chain_id", type=str, help="Chain ID") parser.add_argument("last_irreversible_block_id", type=str, help="Last irreversible block ID") parser.add_argument("contract_owner_account", type=str, help="Cluster contract owner account name") @@ -125,7 +128,7 @@ def main(): connectionPairList = sub('[\s+]', '', args.connection_pair_list) connectionPairList = connectionPairList.rsplit(',') - trxGenLauncher = TransactionGeneratorsLauncher(chainId=args.chain_id, lastIrreversibleBlockId=args.last_irreversible_block_id, + trxGenLauncher = TransactionGeneratorsLauncher(trxGenerator=args.trx_generator, chainId=args.chain_id, lastIrreversibleBlockId=args.last_irreversible_block_id, contractOwnerAccount=args.contract_owner_account, accts=args.accounts, privateKeys=args.priv_keys, trxGenDurationSec=args.trx_gen_duration, logDir=args.log_dir, abiFile=args.abi_file, actionsData=args.actions_data, actionsAuths=args.actions_auths, From d26a930fb8ad3efaed1fcd3d336650cf7da516a5 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 30 Aug 2023 13:35:13 -0500 Subject: [PATCH 2/4] Some cleanup from being included in TestHarness module. --- .../launch_transaction_generators.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tests/TestHarness/launch_transaction_generators.py b/tests/TestHarness/launch_transaction_generators.py index 22951108dc..1285c1523a 100644 --- a/tests/TestHarness/launch_transaction_generators.py +++ b/tests/TestHarness/launch_transaction_generators.py @@ -122,23 +122,3 @@ def parseArgs(): args = parser.parse_args() return args - -def main(): - args = parseArgs() - connectionPairList = sub('[\s+]', '', args.connection_pair_list) - connectionPairList = connectionPairList.rsplit(',') - - trxGenLauncher = TransactionGeneratorsLauncher(trxGenerator=args.trx_generator, chainId=args.chain_id, lastIrreversibleBlockId=args.last_irreversible_block_id, - contractOwnerAccount=args.contract_owner_account, accts=args.accounts, - privateKeys=args.priv_keys, trxGenDurationSec=args.trx_gen_duration, logDir=args.log_dir, - abiFile=args.abi_file, actionsData=args.actions_data, actionsAuths=args.actions_auths, - tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator, - connectionPairList=connectionPairList), - endpointMode=args.endpoint_mode, apiEndpoint=args.api_endpoint) - - - exit_codes = trxGenLauncher.launch() - exit(exit_codes) - -if __name__ == '__main__': - main() From 9f32850ac304baca041ce92713081457010f850d Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 30 Aug 2023 13:50:16 -0500 Subject: [PATCH 3/4] Rename and refactor launch_transactions_generator to TransactionGeneratorsLauncher. --- tests/TestHarness/CMakeLists.txt | 2 +- tests/TestHarness/Cluster.py | 2 +- ...saction_generators.py => TransactionGeneratorsLauncher.py} | 0 tests/TestHarness/__init__.py | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename tests/TestHarness/{launch_transaction_generators.py => TransactionGeneratorsLauncher.py} (100%) diff --git a/tests/TestHarness/CMakeLists.txt b/tests/TestHarness/CMakeLists.txt index e35782ead6..1e00fbf567 100644 --- a/tests/TestHarness/CMakeLists.txt +++ b/tests/TestHarness/CMakeLists.txt @@ -9,7 +9,7 @@ configure_file(queries.py . COPYONLY) configure_file(transactions.py . COPYONLY) configure_file(libc.py . COPYONLY) configure_file(interfaces.py . COPYONLY) -configure_file(launch_transaction_generators.py . COPYONLY) +configure_file(TransactionGeneratorsLauncher.py . COPYONLY) configure_file(logging.py . COPYONLY) configure_file(depresolver.py . COPYONLY) configure_file(launcher.py . COPYONLY) diff --git a/tests/TestHarness/Cluster.py b/tests/TestHarness/Cluster.py index 6d42864d0b..fc0a6b055a 100644 --- a/tests/TestHarness/Cluster.py +++ b/tests/TestHarness/Cluster.py @@ -22,7 +22,7 @@ from .Node import BlockType from .Node import Node from .WalletMgr import WalletMgr -from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig +from .TransactionGeneratorsLauncher import TransactionGeneratorsLauncher, TpsTrxGensConfig from .launcher import cluster_generator try: from .libc import unshare, CLONE_NEWNET diff --git a/tests/TestHarness/launch_transaction_generators.py b/tests/TestHarness/TransactionGeneratorsLauncher.py similarity index 100% rename from tests/TestHarness/launch_transaction_generators.py rename to tests/TestHarness/TransactionGeneratorsLauncher.py diff --git a/tests/TestHarness/__init__.py b/tests/TestHarness/__init__.py index 0341b64a05..c2ba5ad0b3 100644 --- a/tests/TestHarness/__init__.py +++ b/tests/TestHarness/__init__.py @@ -1,4 +1,4 @@ -__all__ = ['Node', 'Cluster', 'WalletMgr', 'launcher', 'logging', 'depresolver', 'testUtils', 'TestHelper', 'queries', 'transactions', 'accounts', 'launch_transaction_generators', 'TransactionGeneratorsLauncher', 'TpsTrxGensConfig', 'core_symbol'] +__all__ = ['Node', 'Cluster', 'WalletMgr', 'launcher', 'logging', 'depresolver', 'testUtils', 'TestHelper', 'queries', 'transactions', 'accounts', 'TransactionGeneratorsLauncher', 'TpsTrxGensConfig', 'core_symbol'] from .Cluster import Cluster from .Node import Node @@ -9,5 +9,5 @@ from .testUtils import Utils from .Node import ReturnType from .TestHelper import TestHelper -from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig +from .TransactionGeneratorsLauncher import TransactionGeneratorsLauncher, TpsTrxGensConfig from .core_symbol import CORE_SYMBOL From 599fda89825f6ba9c02bd791216a7d3a70b2f13f Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Wed, 30 Aug 2023 14:41:20 -0500 Subject: [PATCH 4/4] Update documentation. --- tests/PerformanceHarness/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/PerformanceHarness/README.md b/tests/PerformanceHarness/README.md index 4b267fd1ca..3178aa9e51 100644 --- a/tests/PerformanceHarness/README.md +++ b/tests/PerformanceHarness/README.md @@ -8,7 +8,7 @@ The `PerformanceTest`'s main goal is to measure current peak performance metrics The `PerformanceTestBasic` test performs a single basic performance test that targets a configurable TPS target and, if successful, reports statistics on performance metrics measured during the test. It configures and launches a blockchain test environment, creates wallets and accounts for testing, and configures and launches transaction generators for creating specific transaction load in the ecosystem. Finally it analyzes the performance of the system under the configuration through log analysis and chain queries and produces a [Performance Test Basic Report](#performance-test-basic-report). -The `launch_generators.py` support script provides a means to easily calculate and spawn the number of transaction generator instances to generate a given target TPS, distributing generation load between the instances in a fair manner such that the aggregate load meets the requested test load. +The `TransactionGeneratorsLauncher` provides a means to easily calculate and spawn the number of transaction generator instances to generate a given target TPS, distributing generation load between the instances in a fair manner such that the aggregate load meets the requested test load. The `log_reader.py` support script is used primarily to analyze `nodeos` log files to glean information about generated blocks and transactions within those blocks after a test has concluded. This information is used to produce the performance test report.