From ed975c6425e4acca0057698f97c2c9dc696719dc Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Tue, 29 Nov 2022 09:04:29 -0600 Subject: [PATCH 1/4] Make use of ArgumentParser parents and groups. This will greatly reduce work to bubble up arguments between scripts/modules. Also brings clarity and grouping of arguments by use with group titles and descriptions. Test Helper arguments can now be bubbled up to user scripts via direct access to the ArgumentParser. Performance Test Basic makes use of Test Helper arguments. Perfomance Test makes use of both Performance Test Basic and Test Helper arguments as well. --- tests/TestHarness/TestHelper.py | 75 +++++++------ tests/performance_tests/performance_test.py | 104 +++++++++--------- .../performance_test_basic.py | 93 ++++++++++------ 3 files changed, 153 insertions(+), 119 deletions(-) diff --git a/tests/TestHarness/TestHelper.py b/tests/TestHarness/TestHelper.py index 724f512a22..e9e177cd42 100644 --- a/tests/TestHarness/TestHelper.py +++ b/tests/TestHarness/TestHelper.py @@ -37,87 +37,100 @@ class TestHelper(object): @staticmethod # pylint: disable=too-many-branches # pylint: disable=too-many-statements - def parse_args(includeArgs, applicationSpecificArgs=AppArgs()): + def createArgumentParser(includeArgs, applicationSpecificArgs=AppArgs()) -> argparse.ArgumentParser: """Accepts set of arguments, builds argument parser and returns parse_args() output.""" assert(includeArgs) assert(isinstance(includeArgs, set)) assert(isinstance(applicationSpecificArgs, AppArgs)) - parser = argparse.ArgumentParser(add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-?', action='help', default=argparse.SUPPRESS, + thParser = argparse.ArgumentParser(add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + thGrpTitle = "Test Helper Arguments" + thGrpDescription="Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment." + thGrp = thParser.add_argument_group(title=thGrpTitle, description=thGrpDescription) + thGrp.add_argument('-?', action='help', default=argparse.SUPPRESS, help=argparse._('show this help message and exit')) if "-p" in includeArgs: - parser.add_argument("-p", type=int, help="producing nodes count", default=1) + thGrp.add_argument("-p", type=int, help="producing nodes count", default=1) if "-n" in includeArgs: - parser.add_argument("-n", type=int, help="total nodes", default=0) + thGrp.add_argument("-n", type=int, help="total nodes", default=0) if "-d" in includeArgs: - parser.add_argument("-d", type=int, help="delay between nodes startup", default=1) + thGrp.add_argument("-d", type=int, help="delay between nodes startup", default=1) if "--nodes-file" in includeArgs: - parser.add_argument("--nodes-file", type=str, help="File containing nodes info in JSON format.") + thGrp.add_argument("--nodes-file", type=str, help="File containing nodes info in JSON format.") if "-s" in includeArgs: - parser.add_argument("-s", type=str, help="topology", choices=["mesh"], default="mesh") + thGrp.add_argument("-s", type=str, help="topology", choices=["mesh"], default="mesh") if "-c" in includeArgs: - parser.add_argument("-c", type=str, help="chain strategy", + thGrp.add_argument("-c", type=str, help="chain strategy", choices=[Utils.SyncResyncTag, Utils.SyncReplayTag, Utils.SyncNoneTag, Utils.SyncHardReplayTag], default=Utils.SyncResyncTag) if "--kill-sig" in includeArgs: - parser.add_argument("--kill-sig", type=str, choices=[Utils.SigKillTag, Utils.SigTermTag], help="kill signal.", + thGrp.add_argument("--kill-sig", type=str, choices=[Utils.SigKillTag, Utils.SigTermTag], help="kill signal.", default=Utils.SigKillTag) if "--kill-count" in includeArgs: - parser.add_argument("--kill-count", type=int, help="nodeos instances to kill", default=-1) + thGrp.add_argument("--kill-count", type=int, help="nodeos instances to kill", default=-1) if "--terminate-at-block" in includeArgs: - parser.add_argument("--terminate-at-block", type=int, help="block to terminate on when replaying", default=0) + thGrp.add_argument("--terminate-at-block", type=int, help="block to terminate on when replaying", default=0) if "--seed" in includeArgs: - parser.add_argument("--seed", type=int, help="random seed", default=1) + thGrp.add_argument("--seed", type=int, help="random seed", default=1) if "--host" in includeArgs: - parser.add_argument("-h", "--host", type=str, help="%s host name" % (Utils.EosServerName), + thGrp.add_argument("-h", "--host", type=str, help="%s host name" % (Utils.EosServerName), default=TestHelper.LOCAL_HOST) if "--port" in includeArgs: - parser.add_argument("--port", type=int, help="%s host port" % Utils.EosServerName, + thGrp.add_argument("--port", type=int, help="%s host port" % Utils.EosServerName, default=TestHelper.DEFAULT_PORT) if "--wallet-host" in includeArgs: - parser.add_argument("--wallet-host", type=str, help="%s host" % Utils.EosWalletName, + thGrp.add_argument("--wallet-host", type=str, help="%s host" % Utils.EosWalletName, default=TestHelper.LOCAL_HOST) if "--wallet-port" in includeArgs: - parser.add_argument("--wallet-port", type=int, help="%s port" % Utils.EosWalletName, + thGrp.add_argument("--wallet-port", type=int, help="%s port" % Utils.EosWalletName, default=TestHelper.DEFAULT_WALLET_PORT) if "--prod-count" in includeArgs: - parser.add_argument("-c", "--prod-count", type=int, help="Per node producer count", default=1) + thGrp.add_argument("-c", "--prod-count", type=int, help="Per node producer count", default=1) if "--defproducera_prvt_key" in includeArgs: - parser.add_argument("--defproducera_prvt_key", type=str, help="defproducera private key.") + thGrp.add_argument("--defproducera_prvt_key", type=str, help="defproducera private key.") if "--defproducerb_prvt_key" in includeArgs: - parser.add_argument("--defproducerb_prvt_key", type=str, help="defproducerb private key.") + thGrp.add_argument("--defproducerb_prvt_key", type=str, help="defproducerb private key.") if "--dump-error-details" in includeArgs: - parser.add_argument("--dump-error-details", + thGrp.add_argument("--dump-error-details", help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout", action='store_true') if "--dont-launch" in includeArgs: - parser.add_argument("--dont-launch", help="Don't launch own node. Assume node is already running.", + thGrp.add_argument("--dont-launch", help="Don't launch own node. Assume node is already running.", action='store_true') if "--keep-logs" in includeArgs: - parser.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders, or other test specific log directories, upon test completion", + thGrp.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders, or other test specific log directories, upon test completion", action='store_true') if "-v" in includeArgs: - parser.add_argument("-v", help="verbose logging", action='store_true') + thGrp.add_argument("-v", help="verbose logging", action='store_true') if "--leave-running" in includeArgs: - parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') + thGrp.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') if "--only-bios" in includeArgs: - parser.add_argument("--only-bios", help="Limit testing to bios node.", action='store_true') + thGrp.add_argument("--only-bios", help="Limit testing to bios node.", action='store_true') if "--clean-run" in includeArgs: - parser.add_argument("--clean-run", help="Kill all nodeos and keosd instances", action='store_true') + thGrp.add_argument("--clean-run", help="Kill all nodeos and keosd instances", action='store_true') if "--sanity-test" in includeArgs: - parser.add_argument("--sanity-test", help="Validates nodeos and keosd are in path and can be started up.", action='store_true') + thGrp.add_argument("--sanity-test", help="Validates nodeos and keosd are in path and can be started up.", action='store_true') if "--alternate-version-labels-file" in includeArgs: - parser.add_argument("--alternate-version-labels-file", type=str, help="Provide a file to define the labels that can be used in the test and the path to the version installation associated with that.") + thGrp.add_argument("--alternate-version-labels-file", type=str, help="Provide a file to define the labels that can be used in the test and the path to the version installation associated with that.") + appArgsGrpTitle="Application Specific Arguments" + appArgsGrpdescription="Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment." + appArgsGrp = thParser.add_argument_group(title=appArgsGrpTitle, description=appArgsGrpdescription) for arg in applicationSpecificArgs.args: if arg.type is not None: - parser.add_argument(arg.flag, type=arg.type, help=arg.help, choices=arg.choices, default=arg.default) + appArgsGrp.add_argument(arg.flag, type=arg.type, help=arg.help, choices=arg.choices, default=arg.default) else: - parser.add_argument(arg.flag, help=arg.help, action=arg.action) + appArgsGrp.add_argument(arg.flag, help=arg.help, action=arg.action) + + return thParser + @staticmethod + # pylint: disable=too-many-branches + # pylint: disable=too-many-statements + def parse_args(includeArgs, applicationSpecificArgs=AppArgs()): + parser = TestHelper.createArgumentParser(includeArgs=includeArgs, applicationSpecificArgs=applicationSpecificArgs) args = parser.parse_args() return args diff --git a/tests/performance_tests/performance_test.py b/tests/performance_tests/performance_test.py index 6e11a2c833..ee2b77dd84 100755 --- a/tests/performance_tests/performance_test.py +++ b/tests/performance_tests/performance_test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import copy import math import os @@ -12,7 +13,7 @@ from TestHarness import TestHelper, Utils from TestHarness.TestHelper import AppArgs -from performance_test_basic import PerformanceTestBasic +from performance_test_basic import PerformanceTestBasic, PtbArgumentsHandler from platform import release, system from dataclasses import dataclass, asdict, field from datetime import datetime @@ -470,63 +471,56 @@ def runTest(self): return testSuccessful -def parseArgs(): - appArgs=AppArgs() - appArgs.add(flag="--max-tps-to-test", type=int, help="The max target transfers realistic as ceiling of test range", default=50000) - appArgs.add(flag="--test-iteration-duration-sec", type=int, help="The duration of transfer trx generation for each iteration of the test during the initial search (seconds)", default=150) - appArgs.add(flag="--test-iteration-min-step", type=int, help="The step size determining granularity of tps result during initial search", default=500) - appArgs.add(flag="--final-iterations-duration-sec", type=int, help="The duration of transfer trx generation for each final longer run iteration of the test during the final search (seconds)", default=300) - appArgs.add(flag="--tps-limit-per-generator", type=int, help="Maximum amount of transactions per second a single generator can have.", default=4000) - appArgs.add(flag="--genesis", type=str, help="Path to genesis.json", default="tests/performance_tests/genesis.json") - appArgs.add(flag="--num-blocks-to-prune", type=int, help="The number of potentially non-empty blocks, in addition to leading and trailing size 0 blocks, to prune from the beginning and end of the range of blocks of interest for evaluation.", default=2) - appArgs.add(flag="--signature-cpu-billable-pct", type=int, help="Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50%%", default=0) - appArgs.add(flag="--chain-state-db-size-mb", type=int, help="Maximum size (in MiB) of the chain state database", default=10*1024) - appArgs.add(flag="--chain-threads", type=int, help="Number of worker threads in controller thread pool", default=2) - appArgs.add(flag="--database-map-mode", type=str, help="Database map mode (\"mapped\", \"heap\", or \"locked\"). \ - In \"mapped\" mode database is memory mapped as a file. \ - In \"heap\" mode database is preloaded in to swappable memory and will use huge pages if available. \ - In \"locked\" mode database is preloaded, locked in to memory, and will use huge pages if available.", - choices=["mapped", "heap", "locked"], default="mapped") - appArgs.add(flag="--net-threads", type=int, help="Number of worker threads in net_plugin thread pool", default=2) - appArgs.add(flag="--disable-subjective-billing", type=bool, help="Disable subjective CPU billing for API/P2P transactions", default=True) - appArgs.add(flag="--last-block-time-offset-us", type=int, help="Offset of last block producing time in microseconds. Valid range 0 .. -block_time_interval.", default=0) - appArgs.add(flag="--produce-time-offset-us", type=int, help="Offset of non last block producing time in microseconds. Valid range 0 .. -block_time_interval.", default=0) - appArgs.add(flag="--cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%%", default=100) - appArgs.add(flag="--last-block-cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce last block. Whole number percentages, e.g. 80 for 80%%", default=100) - appArgs.add(flag="--producer-threads", type=int, help="Number of worker threads in producer thread pool", default=2) - appArgs.add(flag="--http-max-response-time-ms", type=int, help="Maximum time for processing a request, -1 for unlimited", default=990000) - appArgs.add_bool(flag="--del-perf-logs", help="Whether to delete performance test specific logs.") - appArgs.add_bool(flag="--del-report", help="Whether to delete overarching performance run report.") - appArgs.add_bool(flag="--del-test-report", help="Whether to save json reports from each test scenario.") - appArgs.add_bool(flag="--quiet", help="Whether to quiet printing intermediate results and reports to stdout") - appArgs.add_bool(flag="--prods-enable-trace-api", help="Determines whether producer nodes should have eosio::trace_api_plugin enabled") - appArgs.add_bool(flag="--skip-tps-test", help="Determines whether to skip the max TPS measurement tests") - appArgs.add(flag="--calc-producer-threads", type=str, help="Determines whether to calculate number of worker threads to use in producer thread pool (\"none\", \"lmax\", or \"full\"). \ - In \"none\" mode, the default, no calculation will be attempted and default configured --producer-threads value will be used. \ - In \"lmax\" mode, producer threads will incrementally be tested until the performance rate ceases to increase with the addition of additional threads. \ - In \"full\" mode producer threads will incrementally be tested from 2..num logical processors, recording each performance and choosing the local max performance (same value as would be discovered in \"lmax\" mode). \ - Useful for graphing the full performance impact of each available thread.", - choices=["none", "lmax", "full"], default="none") - appArgs.add(flag="--calc-chain-threads", type=str, help="Determines whether to calculate number of worker threads to use in chain thread pool (\"none\", \"lmax\", or \"full\"). \ - In \"none\" mode, the default, no calculation will be attempted and default configured --chain-threads value will be used. \ - In \"lmax\" mode, producer threads will incrementally be tested until the performance rate ceases to increase with the addition of additional threads. \ - In \"full\" mode producer threads will incrementally be tested from 2..num logical processors, recording each performance and choosing the local max performance (same value as would be discovered in \"lmax\" mode). \ - Useful for graphing the full performance impact of each available thread.", - choices=["none", "lmax", "full"], default="none") - appArgs.add(flag="--calc-net-threads", type=str, help="Determines whether to calculate number of worker threads to use in net thread pool (\"none\", \"lmax\", or \"full\"). \ - In \"none\" mode, the default, no calculation will be attempted and default configured --net-threads value will be used. \ - In \"lmax\" mode, producer threads will incrementally be tested until the performance rate ceases to increase with the addition of additional threads. \ - In \"full\" mode producer threads will incrementally be tested from 2..num logical processors, recording each performance and choosing the local max performance (same value as would be discovered in \"lmax\" mode). \ - Useful for graphing the full performance impact of each available thread.", - choices=["none", "lmax", "full"], default="none") - args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file" - ,"--dump-error-details","-v","--leave-running" - ,"--clean-run"}, applicationSpecificArgs=appArgs) - return args +class PerfTestArgumentsHandler(object): + @staticmethod + def createArgumentParser(): + ptbArgParser = PtbArgumentsHandler.createBaseArgumentParser() + ptParser = argparse.ArgumentParser(parents=[ptbArgParser], add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + ptGrpTitle="Performance Harness" + ptGrpDescription="Performance Harness testing configuration items." + ptParserGroup = ptParser.add_argument_group(title=ptGrpTitle, description=ptGrpDescription) + ptParserGroup.add_argument("--skip-tps-test", help="Determines whether to skip the max TPS measurement tests", action='store_true') + ptParserGroup.add_argument("--calc-producer-threads", type=str, help="Determines whether to calculate number of worker threads to use in producer thread pool (\"none\", \"lmax\", or \"full\"). \ + In \"none\" mode, the default, no calculation will be attempted and default configured --producer-threads value will be used. \ + In \"lmax\" mode, producer threads will incrementally be tested until the performance rate ceases to increase with the addition of additional threads. \ + In \"full\" mode producer threads will incrementally be tested from 2..num logical processors, recording each performance and choosing the local max performance (same value as would be discovered in \"lmax\" mode). \ + Useful for graphing the full performance impact of each available thread.", + choices=["none", "lmax", "full"], default="none") + ptParserGroup.add_argument("--calc-chain-threads", type=str, help="Determines whether to calculate number of worker threads to use in chain thread pool (\"none\", \"lmax\", or \"full\"). \ + In \"none\" mode, the default, no calculation will be attempted and default configured --chain-threads value will be used. \ + In \"lmax\" mode, producer threads will incrementally be tested until the performance rate ceases to increase with the addition of additional threads. \ + In \"full\" mode producer threads will incrementally be tested from 2..num logical processors, recording each performance and choosing the local max performance (same value as would be discovered in \"lmax\" mode). \ + Useful for graphing the full performance impact of each available thread.", + choices=["none", "lmax", "full"], default="none") + ptParserGroup.add_argument("--calc-net-threads", type=str, help="Determines whether to calculate number of worker threads to use in net thread pool (\"none\", \"lmax\", or \"full\"). \ + In \"none\" mode, the default, no calculation will be attempted and default configured --net-threads value will be used. \ + In \"lmax\" mode, producer threads will incrementally be tested until the performance rate ceases to increase with the addition of additional threads. \ + In \"full\" mode producer threads will incrementally be tested from 2..num logical processors, recording each performance and choosing the local max performance (same value as would be discovered in \"lmax\" mode). \ + Useful for graphing the full performance impact of each available thread.", + choices=["none", "lmax", "full"], default="none") + ptParserGroup.add_argument("--del-test-report", help="Whether to save json reports from each test scenario.", action='store_true') + + ptTpsGrpTitle="TPS Test Config" + ptTpsGrpDescription="TPS Performance Test configuration items." + ptTpsParserGroup = ptParser.add_argument_group(title=ptTpsGrpTitle, description=ptTpsGrpDescription) + + ptTpsParserGroup.add_argument("--max-tps-to-test", type=int, help="The max target transfers realistic as ceiling of test range", default=50000) + ptTpsParserGroup.add_argument("--test-iteration-duration-sec", type=int, help="The duration of transfer trx generation for each iteration of the test during the initial search (seconds)", default=150) + ptTpsParserGroup.add_argument("--test-iteration-min-step", type=int, help="The step size determining granularity of tps result during initial search", default=500) + ptTpsParserGroup.add_argument("--final-iterations-duration-sec", type=int, help="The duration of transfer trx generation for each final longer run iteration of the test during the final search (seconds)", default=300) + + return ptParser + + @staticmethod + def parseArgs(): + ptParser=PerfTestArgumentsHandler.createArgumentParser() + args=ptParser.parse_args() + return args def main(): - args = parseArgs() + args = PerfTestArgumentsHandler.parseArgs() Utils.Debug = args.v testHelperConfig = PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index 1e4610f8a7..b021cd4641 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import os import sys import subprocess @@ -437,42 +438,68 @@ def runTest(self) -> bool: return testSuccessful -def parseArgs(): - appArgs=AppArgs() - appArgs.add(flag="--target-tps", type=int, help="The target transfers per second to send during test", default=8000) - appArgs.add(flag="--tps-limit-per-generator", type=int, help="Maximum amount of transactions per second a single generator can have.", default=4000) - appArgs.add(flag="--test-duration-sec", type=int, help="The duration of transfer trx generation for the test in seconds", default=90) - appArgs.add(flag="--genesis", type=str, help="Path to genesis.json", default="tests/performance_tests/genesis.json") - appArgs.add(flag="--num-blocks-to-prune", type=int, help=("The number of potentially non-empty blocks, in addition to leading and trailing size 0 blocks, " - "to prune from the beginning and end of the range of blocks of interest for evaluation."), default=2) - appArgs.add(flag="--signature-cpu-billable-pct", type=int, help="Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50%%", default=0) - appArgs.add(flag="--chain-state-db-size-mb", type=int, help="Maximum size (in MiB) of the chain state database", default=10*1024) - appArgs.add(flag="--chain-threads", type=int, help="Number of worker threads in controller thread pool", default=2) - appArgs.add(flag="--database-map-mode", type=str, help="Database map mode (\"mapped\", \"heap\", or \"locked\"). \ - In \"mapped\" mode database is memory mapped as a file. \ - In \"heap\" mode database is preloaded in to swappable memory and will use huge pages if available. \ - In \"locked\" mode database is preloaded, locked in to memory, and will use huge pages if available.", - choices=["mapped", "heap", "locked"], default="mapped") - appArgs.add(flag="--net-threads", type=int, help="Number of worker threads in net_plugin thread pool", default=2) - appArgs.add(flag="--disable-subjective-billing", type=bool, help="Disable subjective CPU billing for API/P2P transactions", default=True) - appArgs.add(flag="--last-block-time-offset-us", type=int, help="Offset of last block producing time in microseconds. Valid range 0 .. -block_time_interval.", default=0) - appArgs.add(flag="--produce-time-offset-us", type=int, help="Offset of non last block producing time in microseconds. Valid range 0 .. -block_time_interval.", default=0) - appArgs.add(flag="--cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%%", default=100) - appArgs.add(flag="--last-block-cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce last block. Whole number percentages, e.g. 80 for 80%%", default=100) - appArgs.add(flag="--producer-threads", type=int, help="Number of worker threads in producer thread pool", default=2) - appArgs.add(flag="--http-max-response-time-ms", type=int, help="Maximum time for processing a request, -1 for unlimited", default=990000) - appArgs.add_bool(flag="--del-perf-logs", help="Whether to delete performance test specific logs.") - appArgs.add_bool(flag="--del-report", help="Whether to delete overarching performance run report.") - appArgs.add_bool(flag="--quiet", help="Whether to quiet printing intermediate results and reports to stdout") - appArgs.add_bool(flag="--prods-enable-trace-api", help="Determines whether producer nodes should have eosio::trace_api_plugin enabled") - args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file" - ,"--dump-error-details","-v","--leave-running" - ,"--clean-run"}, applicationSpecificArgs=appArgs) - return args +class PtbArgumentsHandler(object): + @staticmethod + def createBaseArgumentParser(): + testHelperArgParser=TestHelper.createArgumentParser(includeArgs={"-p","-n","-d","-s","--nodes-file" + ,"--dump-error-details","-v","--leave-running" + ,"--clean-run"}) + ptbBaseParser = argparse.ArgumentParser(parents=[testHelperArgParser], add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + ptbBaseGrpTitle="Performance Test Basic Base" + ptbBaseGrpDescription="Performance Test Basic base configuration items." + ptbBaseParserGroup = ptbBaseParser.add_argument_group(title=ptbBaseGrpTitle, description=ptbBaseGrpDescription) + + ptbBaseParserGroup.add_argument("--tps-limit-per-generator", type=int, help="Maximum amount of transactions per second a single generator can have.", default=4000) + ptbBaseParserGroup.add_argument("--genesis", type=str, help="Path to genesis.json", default="tests/performance_tests/genesis.json") + ptbBaseParserGroup.add_argument("--num-blocks-to-prune", type=int, help=("The number of potentially non-empty blocks, in addition to leading and trailing size 0 blocks, " + "to prune from the beginning and end of the range of blocks of interest for evaluation."), default=2) + ptbBaseParserGroup.add_argument("--signature-cpu-billable-pct", type=int, help="Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50%%", default=0) + ptbBaseParserGroup.add_argument("--chain-state-db-size-mb", type=int, help="Maximum size (in MiB) of the chain state database", default=10*1024) + ptbBaseParserGroup.add_argument("--chain-threads", type=int, help="Number of worker threads in controller thread pool", default=2) + ptbBaseParserGroup.add_argument("--database-map-mode", type=str, help="Database map mode (\"mapped\", \"heap\", or \"locked\"). \ + In \"mapped\" mode database is memory mapped as a file. \ + In \"heap\" mode database is preloaded in to swappable memory and will use huge pages if available. \ + In \"locked\" mode database is preloaded, locked in to memory, and will use huge pages if available.", + choices=["mapped", "heap", "locked"], default="mapped") + ptbBaseParserGroup.add_argument("--net-threads", type=int, help="Number of worker threads in net_plugin thread pool", default=2) + ptbBaseParserGroup.add_argument("--disable-subjective-billing", type=bool, help="Disable subjective CPU billing for API/P2P transactions", default=True) + ptbBaseParserGroup.add_argument("--last-block-time-offset-us", type=int, help="Offset of last block producing time in microseconds. Valid range 0 .. -block_time_interval.", default=0) + ptbBaseParserGroup.add_argument("--produce-time-offset-us", type=int, help="Offset of non last block producing time in microseconds. Valid range 0 .. -block_time_interval.", default=0) + ptbBaseParserGroup.add_argument("--cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%%", default=100) + ptbBaseParserGroup.add_argument("--last-block-cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce last block. Whole number percentages, e.g. 80 for 80%%", default=100) + ptbBaseParserGroup.add_argument("--producer-threads", type=int, help="Number of worker threads in producer thread pool", default=2) + ptbBaseParserGroup.add_argument("--http-max-response-time-ms", type=int, help="Maximum time for processing a request, -1 for unlimited", default=990000) + ptbBaseParserGroup.add_argument("--del-perf-logs", help="Whether to delete performance test specific logs.", action='store_true') + ptbBaseParserGroup.add_argument("--del-report", help="Whether to delete overarching performance run report.", action='store_true') + ptbBaseParserGroup.add_argument("--quiet", help="Whether to quiet printing intermediate results and reports to stdout", action='store_true') + ptbBaseParserGroup.add_argument("--prods-enable-trace-api", help="Determines whether producer nodes should have eosio::trace_api_plugin enabled", action='store_true') + return ptbBaseParser + + @staticmethod + def createArgumentParser(): + ptbBaseParser = PtbArgumentsHandler.createBaseArgumentParser() + + ptbParser = argparse.ArgumentParser(parents=[ptbBaseParser], add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + ptbGrpTitle="Performance Test Basic Single Test" + ptbGrpDescription="Performance Test Basic single test configuration items. Useful for running a single test directly. \ + These items may not be directly configurable from higher level scripts as the scripts themselves may configure these internally." + ptbParserGroup = ptbBaseParser.add_argument_group(title=ptbGrpTitle, description=ptbGrpDescription) + + ptbParserGroup.add_argument("--target-tps", type=int, help="The target transfers per second to send during test", default=8000) + ptbParserGroup.add_argument("--test-duration-sec", type=int, help="The duration of transfer trx generation for the test in seconds", default=90) + return ptbParser + + @staticmethod + def parseArgs(): + ptbParser=PtbArgumentsHandler.createArgumentParser() + args=ptbParser.parse_args() + return args def main(): - args = parseArgs() + args = PtbArgumentsHandler.parseArgs() Utils.Debug = args.v testHelperConfig = PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, From ee1795570de7e6d936c3f42273ebc79ef76a687d Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Tue, 29 Nov 2022 09:09:07 -0600 Subject: [PATCH 2/4] Don't print Application Specific Arguments group if there are none. --- tests/TestHarness/TestHelper.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/TestHarness/TestHelper.py b/tests/TestHarness/TestHelper.py index e9e177cd42..01874c6be0 100644 --- a/tests/TestHarness/TestHelper.py +++ b/tests/TestHarness/TestHelper.py @@ -115,14 +115,15 @@ def createArgumentParser(includeArgs, applicationSpecificArgs=AppArgs()) -> argp if "--alternate-version-labels-file" in includeArgs: thGrp.add_argument("--alternate-version-labels-file", type=str, help="Provide a file to define the labels that can be used in the test and the path to the version installation associated with that.") - appArgsGrpTitle="Application Specific Arguments" - appArgsGrpdescription="Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment." - appArgsGrp = thParser.add_argument_group(title=appArgsGrpTitle, description=appArgsGrpdescription) - for arg in applicationSpecificArgs.args: - if arg.type is not None: - appArgsGrp.add_argument(arg.flag, type=arg.type, help=arg.help, choices=arg.choices, default=arg.default) - else: - appArgsGrp.add_argument(arg.flag, help=arg.help, action=arg.action) + if len(applicationSpecificArgs.args) > 0: + appArgsGrpTitle="Application Specific Arguments" + appArgsGrpdescription="Test Helper configuration items used to configure and spin up the regression test framework and blockchain environment." + appArgsGrp = thParser.add_argument_group(title=appArgsGrpTitle, description=appArgsGrpdescription) + for arg in applicationSpecificArgs.args: + if arg.type is not None: + appArgsGrp.add_argument(arg.flag, type=arg.type, help=arg.help, choices=arg.choices, default=arg.default) + else: + appArgsGrp.add_argument(arg.flag, help=arg.help, action=arg.action) return thParser From b6b6bcf767e74cc252d57f556ec58ab0d91abbb7 Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Tue, 29 Nov 2022 09:11:01 -0600 Subject: [PATCH 3/4] Clarify TPS Test Config is part of Performance Harness. --- tests/performance_tests/performance_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance_tests/performance_test.py b/tests/performance_tests/performance_test.py index ee2b77dd84..ecdb068411 100755 --- a/tests/performance_tests/performance_test.py +++ b/tests/performance_tests/performance_test.py @@ -501,7 +501,7 @@ def createArgumentParser(): choices=["none", "lmax", "full"], default="none") ptParserGroup.add_argument("--del-test-report", help="Whether to save json reports from each test scenario.", action='store_true') - ptTpsGrpTitle="TPS Test Config" + ptTpsGrpTitle="Performance Harness - TPS Test Config" ptTpsGrpDescription="TPS Performance Test configuration items." ptTpsParserGroup = ptParser.add_argument_group(title=ptTpsGrpTitle, description=ptTpsGrpDescription) From bd4a525f2f1438530c8063a3a31940352d237f7a Mon Sep 17 00:00:00 2001 From: Peter Oschwald Date: Tue, 29 Nov 2022 09:38:24 -0600 Subject: [PATCH 4/4] Fix performance_test_basic.py parser. --- tests/performance_tests/performance_test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index b021cd4641..264045660d 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -485,7 +485,7 @@ def createArgumentParser(): ptbGrpTitle="Performance Test Basic Single Test" ptbGrpDescription="Performance Test Basic single test configuration items. Useful for running a single test directly. \ These items may not be directly configurable from higher level scripts as the scripts themselves may configure these internally." - ptbParserGroup = ptbBaseParser.add_argument_group(title=ptbGrpTitle, description=ptbGrpDescription) + ptbParserGroup = ptbParser.add_argument_group(title=ptbGrpTitle, description=ptbGrpDescription) ptbParserGroup.add_argument("--target-tps", type=int, help="The target transfers per second to send during test", default=8000) ptbParserGroup.add_argument("--test-duration-sec", type=int, help="The duration of transfer trx generation for the test in seconds", default=90)