From db731420950bc396c4c5a9cdb97e926b0705ae70 Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Wed, 22 Feb 2023 18:24:25 -0600 Subject: [PATCH 1/6] Expose a number of nodeos options via the CLI in performance_test_basic. Also hardcode some others. --- tests/TestHarness/Cluster.py | 11 +++++- tests/performance_tests/README.md | 38 +++++++++++++++++-- .../performance_test_basic.py | 27 +++++++++---- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/tests/TestHarness/Cluster.py b/tests/TestHarness/Cluster.py index c0d66391d6..3d6e29b5bd 100644 --- a/tests/TestHarness/Cluster.py +++ b/tests/TestHarness/Cluster.py @@ -257,8 +257,15 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me cmdArr=cmd.split() if self.staging: cmdArr.append("--nogen") - - nodeosArgs="--max-transaction-time -1 --abi-serializer-max-time-ms 990000 --p2p-max-nodes-per-host %d --max-clients %d" % (maximumP2pPerHost, maximumClients) + nodeosArgs="" + if extraNodeosArgs.find("--max-transaction-time") == -1: + nodeosArgs += " --max-transaction-time -1" + if extraNodeosArgs.find("--abi-serializer-max-time-ms") == -1: + nodeosArgs += " --abi-serializer-max-time-ms 990000" + if extraNodeosArgs.find("--p2p-max-nodes-per-host") == -1: + nodeosArgs += f" --p2p-max-nodes-per-host {maximumP2pPerHost}" + if extraNodeosArgs.find("--max-clients") == -1: + nodeosArgs += f" --max-clients {maximumClients}" if not self.walletd: nodeosArgs += " --plugin eosio::wallet_api_plugin" if Utils.Debug: diff --git a/tests/performance_tests/README.md b/tests/performance_tests/README.md index 9919bae1d2..543a4c90f7 100644 --- a/tests/performance_tests/README.md +++ b/tests/performance_tests/README.md @@ -226,8 +226,6 @@ The Performance Harness main script `performance_test.py` can be configured usin (default: 2) * `--signature-cpu-billable-pct SIGNATURE_CPU_BILLABLE_PCT` Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50% (default: 0) -* `--chain-state-db-size-mb CHAIN_STATE_DB_SIZE_MB` - Maximum size (in MiB) of the chain state database (default: 10240) * `--chain-threads CHAIN_THREADS` Number of worker threads in controller thread pool (default: 3) * `--database-map-mode {mapped,heap,locked}` @@ -279,6 +277,23 @@ The Performance Harness main script `performance_test.py` can be configured usin * `--contract-dir` Path to contract dir * `--wasm-file` WASM file name for contract * `--abi-file` ABI file name for contract +* `--wasm-runtime RUNTIME` + Override default WASM runtime ("eos-vm-jit", "eos-vm") + "eos-vm-jit" : A WebAssembly runtime that compiles WebAssembly code to native x86 code prior to + execution. "eos-vm" : A WebAssembly interpreter. (default: eos-vm-jit) +* `--contracts-console` print contract's output to console (default: False) +* `--eos-vm-oc-cache-size-mb CACHE_SIZE_MiB` + Maximum size (in MiB) of the EOS VM OC code cache (default: 1024) +* `--eos-vm-oc-compile-threads COMPILE_THREADS` + Number of threads to use for EOS VM OC tier-up (default: 1) +* `--eos-vm-oc-enable` + Enable EOS VM OC tier-up runtime (default: False) +* `--block-log-retain-blocks BLOCKS_TO_RETAIN` + If set to greater than 0, periodically prune the block log to + store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log; + block log file is removed after startup. (default: None) +* `--http-threads HTTP_THREADS` + Number of worker threads in http thread pool (default: 2) ### Support Scripts @@ -314,8 +329,6 @@ The following scripts are typically used by the Performance Harness main script of the range of blocks of interest for evaluation. (default: 2) * `--signature-cpu-billable-pct SIGNATURE_CPU_BILLABLE_PCT` Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50% (default: 0) -* `--chain-state-db-size-mb CHAIN_STATE_DB_SIZE_MB` - Maximum size (in MiB) of the chain state database (default: 10240) * `--chain-threads CHAIN_THREADS` Number of worker threads in controller thread pool (default: 3) * `--database-map-mode {mapped,heap,locked}` @@ -352,6 +365,23 @@ The following scripts are typically used by the Performance Harness main script * `--contract-dir` Path to contract dir * `--wasm-file` WASM file name for contract * `--abi-file` ABI file name for contract +* `--wasm-runtime RUNTIME` + Override default WASM runtime ("eos-vm-jit", "eos-vm") + "eos-vm-jit" : A WebAssembly runtime that compiles WebAssembly code to native x86 code prior to + execution. "eos-vm" : A WebAssembly interpreter. (default: eos-vm-jit) +* `--contracts-console` print contract's output to console (default: False) +* `--eos-vm-oc-cache-size-mb CACHE_SIZE_MiB` + Maximum size (in MiB) of the EOS VM OC code cache (default: 1024) +* `--eos-vm-oc-compile-threads COMPILE_THREADS` + Number of threads to use for EOS VM OC tier-up (default: 1) +* `--eos-vm-oc-enable` + Enable EOS VM OC tier-up runtime (default: False) +* `--block-log-retain-blocks BLOCKS_TO_RETAIN` + If set to greater than 0, periodically prune the block log to + store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log; + block log file is removed after startup. (default: None) +* `--http-threads HTTP_THREADS` + Number of worker threads in http thread pool (default: 2) #### Launch Transaction Generators (TestHarness) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index a8c6941d05..c2f1581c42 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -527,7 +527,6 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--num-blocks-to-prune", type=int, help=("The number of potentially non-empty blocks, in addition to leading and trailing size 0 blocks, " "to prune from the beginning and end of the range of blocks of interest for evaluation."), default=2) ptbBaseParserGroup.add_argument("--signature-cpu-billable-pct", type=int, help="Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50%%", default=0) - ptbBaseParserGroup.add_argument("--chain-state-db-size-mb", type=int, help="Maximum size (in MiB) of the chain state database", default=10*1024) ptbBaseParserGroup.add_argument("--chain-threads", type=int, help="Number of worker threads in controller thread pool", default=2) ptbBaseParserGroup.add_argument("--database-map-mode", type=str, help="Database map mode (\"mapped\", \"heap\", or \"locked\"). \ In \"mapped\" mode database is memory mapped as a file. \ @@ -557,6 +556,18 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--contract-dir", type=str, help="Path to contract dir", default="unittests/contracts/eosio.system") ptbBaseParserGroup.add_argument("--wasm-file", type=str, help="WASM file name for contract", default="eosio.system.wasm") ptbBaseParserGroup.add_argument("--abi-file", type=str, help="ABI file name for contract", default="eosio.system.abi") + ptbBaseParserGroup.add_argument("--wasm-runtime", type=str, help="Override default WASM runtime (\"eos-vm-jit\", \"eos-vm\")\ + \"eos-vm-jit\" : A WebAssembly runtime that compiles WebAssembly code to native x86 code prior to\ + execution. \"eos-vm\" : A WebAssembly interpreter.", default="eos-vm-jit") + ptbBaseParserGroup.add_argument("--contracts-console", type=bool, help="print contract's output to console", default=False) + ptbBaseParserGroup.add_argument("--eos-vm-oc-cache-size-mb", type=int, help="Maximum size (in MiB) of the EOS VM OC code cache", default=1024) + ptbBaseParserGroup.add_argument("--eos-vm-oc-compile-threads", type=int, help="Number of threads to use for EOS VM OC tier-up", default=1) + ptbBaseParserGroup.add_argument("--eos-vm-oc-enable", type=bool, help="Enable EOS VM OC tier-up runtime", default=False) + ptbBaseParserGroup.add_argument("--block-log-retain-blocks", type=int, help="If set to greater than 0, periodically prune the block log to\ + store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log;\ + block log file is removed after startup.", default=None) + ptbBaseParserGroup.add_argument("--http-threads", type=int, help="Number of worker threads in http thread pool", default=2) + return ptbBaseParser @staticmethod @@ -580,6 +591,8 @@ def createArgumentParser(): def parseArgs(): ptbParser=PtbArgumentsHandler.createArgumentParser() args=ptbParser.parse_args() + if args.contracts_console: + print("Enabling contracts-console will not print anything unless debug level is 'debug' or higher.") return args def main(): @@ -590,8 +603,7 @@ def main(): testHelperConfig = PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, dumpErrorDetails=args.dump_error_details, delay=args.d, nodesFile=args.nodes_file, verbose=args.v) - chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, chainStateDbSizeMb=args.chain_state_db_size_mb, - chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode) + chainPluginArgs = ChainPluginArgs(chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) lbto = args.last_block_time_offset_us lbcep = args.last_block_cpu_effort_percent @@ -602,11 +614,12 @@ def main(): producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, lastBlockTimeOffsetUs=lbto, produceTimeOffsetUs=args.produce_time_offset_us, cpuEffortPercent=args.cpu_effort_percent, lastBlockCpuEffortPercent=lbcep, - producerThreads=args.producer_threads) - httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms) - netPluginArgs = NetPluginArgs(netThreads=args.net_threads) + producerThreads=args.producer_threads, maxTransactionTime=-1) + httpPluginArgs = HttpPluginArgs(httpMaxBytesInFlightMb=-1, httpMaxResponseTimeMs=-1) + netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) + resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=True) ENA = PerformanceTestBasic.ClusterConfig.ExtraNodeosArgs - extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs) + extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, resourceMonitorPluginArgs=resourceMonitorPluginArgs) SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract specifiedContract=SC(accountName=args.account_name, ownerPublicKey=args.owner_public_key, activePublicKey=args.active_public_key, contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file) From dcd8bc6df16b25067571682e66258d087e446f3c Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Thu, 23 Feb 2023 14:52:42 -0600 Subject: [PATCH 2/6] Restore accidentally removed config option. Added new option and updated README. Use choices for wasm-runtime. Add new options to the *Args constructors in both performance tests. --- tests/TestHarness/Cluster.py | 10 ++++---- tests/performance_tests/README.md | 10 ++++++-- tests/performance_tests/performance_test.py | 21 ++++++++++------ .../performance_test_basic.py | 25 +++++++++++++------ 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/tests/TestHarness/Cluster.py b/tests/TestHarness/Cluster.py index 004cd40d66..edba218d8b 100644 --- a/tests/TestHarness/Cluster.py +++ b/tests/TestHarness/Cluster.py @@ -258,17 +258,17 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me if self.staging: cmdArr.append("--nogen") nodeosArgs="" - if extraNodeosArgs.find("--max-transaction-time") == -1: + if "--max-transaction-time" not in extraNodeosArgs: nodeosArgs += " --max-transaction-time -1" - if extraNodeosArgs.find("--abi-serializer-max-time-ms") == -1: + if "--abi-serializer-max-time-ms" not in extraNodeosArgs: nodeosArgs += " --abi-serializer-max-time-ms 990000" - if extraNodeosArgs.find("--p2p-max-nodes-per-host") == -1: + if "--p2p-max-nodes-per-host" not in extraNodeosArgs: nodeosArgs += f" --p2p-max-nodes-per-host {maximumP2pPerHost}" - if extraNodeosArgs.find("--max-clients") == -1: + if "--max-clients" not in extraNodeosArgs: nodeosArgs += f" --max-clients {maximumClients}" if not self.walletd: nodeosArgs += " --plugin eosio::wallet_api_plugin" - if Utils.Debug: + if Utils.Debug and "--contracts-console" not in extraNodeosArgs: nodeosArgs += " --contracts-console" if PFSetupPolicy.hasPreactivateFeature(pfSetupPolicy): nodeosArgs += " --plugin eosio::producer_api_plugin" diff --git a/tests/performance_tests/README.md b/tests/performance_tests/README.md index 9c7357704a..0b3ce2a951 100644 --- a/tests/performance_tests/README.md +++ b/tests/performance_tests/README.md @@ -248,7 +248,10 @@ The Performance Harness main script `performance_test.py` can be configured usin * `--producer-threads PRODUCER_THREADS` Number of worker threads in producer thread pool (default: 6) * `--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS` - Maximum time for processing a request, -1 for unlimited (default: 990000) + Maximum time for processing a request, -1 for unlimited (default: -1) +* `--http-max-bytes-inflight-mb HTTP_MAX_BYTES_IN FLIGHT` + Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. + 429 error response when exceeded. (default: -1) * `--del-perf-logs` Whether to delete performance test specific logs. (default: False) * `--del-report` Whether to delete overarching performance run report. (default: False) * `--del-test-report` Whether to save json reports from each test scenario. (default: False) @@ -349,7 +352,10 @@ The following scripts are typically used by the Performance Harness main script * `--producer-threads PRODUCER_THREADS` Number of worker threads in producer thread pool (default: 6) * `--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS` - Maximum time for processing a request, -1 for unlimited (default: 990000) + Maximum time for processing a request, -1 for unlimited (default: -1) +* `--http-max-bytes-inflight-mb HTTP_MAX_BYTES_IN FLIGHT` + Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. + 429 error response when exceeded. (default: -1) * `--del-perf-logs` Whether to delete performance test specific logs. (default: False) * `--del-report` Whether to delete overarching performance run report. (default: False) * `--quiet` Whether to quiet printing intermediate results and reports to stdout (default: False) diff --git a/tests/performance_tests/performance_test.py b/tests/performance_tests/performance_test.py index c99132ad2a..e20c0ca710 100755 --- a/tests/performance_tests/performance_test.py +++ b/tests/performance_tests/performance_test.py @@ -11,7 +11,7 @@ from pathlib import Path, PurePath sys.path.append(str(PurePath(PurePath(Path(__file__).absolute()).parent).parent)) -from NodeosPluginArgs import ChainPluginArgs, HttpPluginArgs, NetPluginArgs, ProducerPluginArgs +from NodeosPluginArgs import ChainPluginArgs, HttpPluginArgs, NetPluginArgs, ProducerPluginArgs, ResourceMonitorPluginArgs from TestHarness import TestHelper, Utils, Account from performance_test_basic import PerformanceTestBasic, PtbArgumentsHandler from platform import release, system @@ -498,15 +498,22 @@ def main(): verbose=args.v) ENA = PerformanceTestBasic.ClusterConfig.ExtraNodeosArgs - chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, chainStateDbSizeMb=args.chain_state_db_size_mb, - chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode) + chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, + chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, + wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, + eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, + eosVmOcEnable=args.eos_vm_oc_enable, blockLogRetainBlocks=args.block_log_retain_blocks, + abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, lastBlockTimeOffsetUs=args.last_block_time_offset_us, produceTimeOffsetUs=args.produce_time_offset_us, cpuEffortPercent=args.cpu_effort_percent, lastBlockCpuEffortPercent=args.last_block_cpu_effort_percent, - producerThreads=args.producer_threads) - httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms) - netPluginArgs = NetPluginArgs(netThreads=args.net_threads) - extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs) + producerThreads=args.producer_threads, maxTransactionTime=-1) + httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms, httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, + httpThreads=args.http_threads) + netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) + resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=True) + extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, + resourceMonitorPluginArgs=resourceMonitorPluginArgs) testClusterConfig = PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, specifiedContract=PerformanceTestBasic.ClusterConfig.SpecifiedContract(account=Account(args.account_name), diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index eb3a38758d..57c6024f2c 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -541,7 +541,9 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce block. Whole number percentages, e.g. 80 for 80%%", default=100) ptbBaseParserGroup.add_argument("--last-block-cpu-effort-percent", type=int, help="Percentage of cpu block production time used to produce last block. Whole number percentages, e.g. 80 for 80%%", default=100) ptbBaseParserGroup.add_argument("--producer-threads", type=int, help="Number of worker threads in producer thread pool", default=2) - ptbBaseParserGroup.add_argument("--http-max-response-time-ms", type=int, help="Maximum time for processing a request, -1 for unlimited", default=990000) + ptbBaseParserGroup.add_argument("--http-max-response-time-ms", type=int, help="Maximum time for processing a request, -1 for unlimited", default=-1) + ptbBaseParserGroup.add_argument("--http-max-bytes-in-flight-mb", type=int, help="Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. 429\ + error response when exceeded.", default=-1) ptbBaseParserGroup.add_argument("--del-perf-logs", help="Whether to delete performance test specific logs.", action='store_true') ptbBaseParserGroup.add_argument("--del-report", help="Whether to delete overarching performance run report.", action='store_true') ptbBaseParserGroup.add_argument("--quiet", help="Whether to quiet printing intermediate results and reports to stdout", action='store_true') @@ -553,7 +555,8 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--abi-file", type=str, help="ABI file name for contract", default="eosio.system.abi") ptbBaseParserGroup.add_argument("--wasm-runtime", type=str, help="Override default WASM runtime (\"eos-vm-jit\", \"eos-vm\")\ \"eos-vm-jit\" : A WebAssembly runtime that compiles WebAssembly code to native x86 code prior to\ - execution. \"eos-vm\" : A WebAssembly interpreter.", default="eos-vm-jit") + execution. \"eos-vm\" : A WebAssembly interpreter.", + choices=["eos-vm-jit", "eos-vm", "eos-vm-jit"], default="eos-vm-jit") ptbBaseParserGroup.add_argument("--contracts-console", type=bool, help="print contract's output to console", default=False) ptbBaseParserGroup.add_argument("--eos-vm-oc-cache-size-mb", type=int, help="Maximum size (in MiB) of the EOS VM OC code cache", default=1024) ptbBaseParserGroup.add_argument("--eos-vm-oc-compile-threads", type=int, help="Number of threads to use for EOS VM OC tier-up", default=1) @@ -586,8 +589,6 @@ def createArgumentParser(): def parseArgs(): ptbParser=PtbArgumentsHandler.createArgumentParser() args=ptbParser.parse_args() - if args.contracts_console: - print("Enabling contracts-console will not print anything unless debug level is 'debug' or higher.") return args def main(): @@ -598,7 +599,12 @@ def main(): testHelperConfig = PerformanceTestBasic.TestHelperConfig(killAll=args.clean_run, dontKill=args.leave_running, keepLogs=not args.del_perf_logs, dumpErrorDetails=args.dump_error_details, delay=args.d, nodesFile=args.nodes_file, verbose=args.v) - chainPluginArgs = ChainPluginArgs(chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) + chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, + chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, + wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, + eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, + eosVmOcEnable=args.eos_vm_oc_enable, blockLogRetainBlocks=args.block_log_retain_blocks, + abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) lbto = args.last_block_time_offset_us lbcep = args.last_block_cpu_effort_percent @@ -610,17 +616,22 @@ def main(): lastBlockTimeOffsetUs=lbto, produceTimeOffsetUs=args.produce_time_offset_us, cpuEffortPercent=args.cpu_effort_percent, lastBlockCpuEffortPercent=lbcep, producerThreads=args.producer_threads, maxTransactionTime=-1) - httpPluginArgs = HttpPluginArgs(httpMaxBytesInFlightMb=-1, httpMaxResponseTimeMs=-1) + httpPluginArgs = HttpPluginArgs(httpMaxResponseTimeMs=args.http_max_response_time_ms, httpMaxBytesInFlightMb=args.http_max_bytes_in_flight_mb, + httpThreads=args.http_threads) netPluginArgs = NetPluginArgs(netThreads=args.net_threads, maxClients=0) resourceMonitorPluginArgs = ResourceMonitorPluginArgs(resourceMonitorNotShutdownOnThresholdExceeded=True) ENA = PerformanceTestBasic.ClusterConfig.ExtraNodeosArgs - extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, resourceMonitorPluginArgs=resourceMonitorPluginArgs) + extraNodeosArgs = ENA(chainPluginArgs=chainPluginArgs, httpPluginArgs=httpPluginArgs, producerPluginArgs=producerPluginArgs, netPluginArgs=netPluginArgs, + resourceMonitorPluginArgs=resourceMonitorPluginArgs) SC = PerformanceTestBasic.ClusterConfig.SpecifiedContract specifiedContract=SC(contractDir=args.contract_dir, wasmFile=args.wasm_file, abiFile=args.abi_file, account=Account(args.account_name)) testClusterConfig = PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, nodeosVers=Utils.getNodeosVersion().split('.')[0]) + if args.contracts_console and testClusterConfig.loggingLevel != "debug" and testClusterConfig.loggingLevel != "all": + print("Enabling contracts-console will not print anything unless debug level is 'debug' or higher." + f" Current debug level is: {testClusterConfig.loggingLevel}") ptbConfig = PerformanceTestBasic.PtbConfig(targetTps=args.target_tps, testTrxGenDurationSec=args.test_duration_sec, tpsLimitPerGenerator=args.tps_limit_per_generator, numAddlBlocksToPrune=args.num_blocks_to_prune, logDirRoot=".", delReport=args.del_report, quiet=args.quiet, delPerfLogs=args.del_perf_logs, printMissingTransactions=args.print_missing_transactions, From 481811969efa19bd9215c261afc5b7447ee82aa6 Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Thu, 23 Feb 2023 14:55:50 -0600 Subject: [PATCH 3/6] fix some README issues in performance test --- tests/performance_tests/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance_tests/README.md b/tests/performance_tests/README.md index 0b3ce2a951..7aaac9a321 100644 --- a/tests/performance_tests/README.md +++ b/tests/performance_tests/README.md @@ -249,7 +249,7 @@ The Performance Harness main script `performance_test.py` can be configured usin Number of worker threads in producer thread pool (default: 6) * `--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS` Maximum time for processing a request, -1 for unlimited (default: -1) -* `--http-max-bytes-inflight-mb HTTP_MAX_BYTES_IN FLIGHT` +* `--http-max-bytes-in-flight-mb HTTP_MAX_IN_FLIGHT_BYTES` Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. 429 error response when exceeded. (default: -1) * `--del-perf-logs` Whether to delete performance test specific logs. (default: False) @@ -353,7 +353,7 @@ The following scripts are typically used by the Performance Harness main script Number of worker threads in producer thread pool (default: 6) * `--http-max-response-time-ms HTTP_MAX_RESPONSE_TIME_MS` Maximum time for processing a request, -1 for unlimited (default: -1) -* `--http-max-bytes-inflight-mb HTTP_MAX_BYTES_IN FLIGHT` +* `--http-max-bytes-in-flight-mb HTTP_MAX_IN_FLIGHT_BYTES` Maximum size in megabytes http_plugin should use for processing http requests. -1 for unlimited. 429 error response when exceeded. (default: -1) * `--del-perf-logs` Whether to delete performance test specific logs. (default: False) From 1ce9036357e409fa47096941d94f4d1fb8760eb2 Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Mon, 27 Feb 2023 16:36:38 -0600 Subject: [PATCH 4/6] change enable args in performance_test_basic to be store_true instead of bools. Put eos-vm-oc-enable in ptb only onto non producer nodes. Remove duplicate choice in wasm-runtime --- .../performance_tests/performance_test_basic.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index 57c6024f2c..6ed9eb7777 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -90,6 +90,7 @@ class SpecifiedContract: nodeosVers: str = "" specificExtraNodeosArgs: dict = field(default_factory=dict) _totalNodes: int = 2 + nonprodsEnableEosVmOcEnable: bool = False def log_transactions(self, trxDataFile, block): for trx in block['payload']['transactions']: @@ -101,6 +102,8 @@ def __post_init__(self): self._totalNodes = self.pnodes + 1 if self.totalNodes <= self.pnodes else self.totalNodes if not self.prodsEnableTraceApi: self.specificExtraNodeosArgs.update({f"{node}" : "--plugin eosio::trace_api_plugin" for node in range(self.pnodes, self._totalNodes)}) + if self.nonprodsEnableEosVmOcEnable: + self.specificExtraNodeosArgs.update({f"{node}" : "--eos-vm-oc-enable" for node in range(self.pnodes, self._totalNodes)}) assert self.nodeosVers != "v1" and self.nodeosVers != "v0", f"nodeos version {Utils.getNodeosVersion().split('.')[0]} is unsupported by performance test" if self.nodeosVers == "v2": self.fetchBlock = lambda node, blockNum: node.processUrllibRequest("chain", "get_block", {"block_num_or_id":blockNum}, silentErrors=False, exitOnError=True) @@ -548,7 +551,7 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--del-report", help="Whether to delete overarching performance run report.", action='store_true') ptbBaseParserGroup.add_argument("--quiet", help="Whether to quiet printing intermediate results and reports to stdout", action='store_true') ptbBaseParserGroup.add_argument("--prods-enable-trace-api", help="Determines whether producer nodes should have eosio::trace_api_plugin enabled", action='store_true') - ptbBaseParserGroup.add_argument("--print-missing-transactions", type=bool, help="Toggles if missing transactions are be printed upon test completion.", default=False) + ptbBaseParserGroup.add_argument("--print-missing-transactions", help="Toggles if missing transactions are be printed upon test completion.", action='store_true') ptbBaseParserGroup.add_argument("--account-name", type=str, help="Name of the account to create and assign a contract to", default="eosio") ptbBaseParserGroup.add_argument("--contract-dir", type=str, help="Path to contract dir", default="unittests/contracts/eosio.system") ptbBaseParserGroup.add_argument("--wasm-file", type=str, help="WASM file name for contract", default="eosio.system.wasm") @@ -556,11 +559,11 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--wasm-runtime", type=str, help="Override default WASM runtime (\"eos-vm-jit\", \"eos-vm\")\ \"eos-vm-jit\" : A WebAssembly runtime that compiles WebAssembly code to native x86 code prior to\ execution. \"eos-vm\" : A WebAssembly interpreter.", - choices=["eos-vm-jit", "eos-vm", "eos-vm-jit"], default="eos-vm-jit") - ptbBaseParserGroup.add_argument("--contracts-console", type=bool, help="print contract's output to console", default=False) + choices=["eos-vm-jit", "eos-vm"], default="eos-vm-jit") + ptbBaseParserGroup.add_argument("--contracts-console", help="print contract's output to console", action='store_true') ptbBaseParserGroup.add_argument("--eos-vm-oc-cache-size-mb", type=int, help="Maximum size (in MiB) of the EOS VM OC code cache", default=1024) ptbBaseParserGroup.add_argument("--eos-vm-oc-compile-threads", type=int, help="Number of threads to use for EOS VM OC tier-up", default=1) - ptbBaseParserGroup.add_argument("--eos-vm-oc-enable", type=bool, help="Enable EOS VM OC tier-up runtime", default=False) + ptbBaseParserGroup.add_argument("--eos-vm-oc-enable", help="Enable EOS VM OC tier-up runtime", action='store_true') ptbBaseParserGroup.add_argument("--block-log-retain-blocks", type=int, help="If set to greater than 0, periodically prune the block log to\ store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log;\ block log file is removed after startup.", default=None) @@ -602,8 +605,7 @@ def main(): chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, - eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, - eosVmOcEnable=args.eos_vm_oc_enable, blockLogRetainBlocks=args.block_log_retain_blocks, + eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, blockLogRetainBlocks=args.block_log_retain_blocks, abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) lbto = args.last_block_time_offset_us @@ -628,7 +630,8 @@ def main(): testClusterConfig = PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, - nodeosVers=Utils.getNodeosVersion().split('.')[0]) + nodeosVers=Utils.getNodeosVersion().split('.')[0], nonprodsEnableEosVmOcEnable=args.eos_vm_oc_enable) + if args.contracts_console and testClusterConfig.loggingLevel != "debug" and testClusterConfig.loggingLevel != "all": print("Enabling contracts-console will not print anything unless debug level is 'debug' or higher." f" Current debug level is: {testClusterConfig.loggingLevel}") From 0d30973b85b9b2667f57761e5d6cbc42478be1fa Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Tue, 28 Feb 2023 14:39:06 -0600 Subject: [PATCH 5/6] rename eos-vm-oc-enable in performance test to represent it only affecting non producer nodes. Resolve related test failure. --- tests/performance_tests/README.md | 8 ++++---- tests/performance_tests/performance_test.py | 2 +- .../performance_tests/performance_test_basic.py | 17 ++++++++++------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/performance_tests/README.md b/tests/performance_tests/README.md index 7aaac9a321..6bc9a24c44 100644 --- a/tests/performance_tests/README.md +++ b/tests/performance_tests/README.md @@ -287,8 +287,8 @@ The Performance Harness main script `performance_test.py` can be configured usin Maximum size (in MiB) of the EOS VM OC code cache (default: 1024) * `--eos-vm-oc-compile-threads COMPILE_THREADS` Number of threads to use for EOS VM OC tier-up (default: 1) -* `--eos-vm-oc-enable` - Enable EOS VM OC tier-up runtime (default: False) +* `--non-prods-eos-vm-oc-enable` + Enable EOS VM OC tier-up runtime on non producer nodes (default: False) * `--block-log-retain-blocks BLOCKS_TO_RETAIN` If set to greater than 0, periodically prune the block log to store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log; @@ -376,8 +376,8 @@ The following scripts are typically used by the Performance Harness main script Maximum size (in MiB) of the EOS VM OC code cache (default: 1024) * `--eos-vm-oc-compile-threads COMPILE_THREADS` Number of threads to use for EOS VM OC tier-up (default: 1) -* `--eos-vm-oc-enable` - Enable EOS VM OC tier-up runtime (default: False) +* `--non-prods-eos-vm-oc-enable` + Enable EOS VM OC tier-up runtime on non producer nodes (default: False) * `--block-log-retain-blocks BLOCKS_TO_RETAIN` If set to greater than 0, periodically prune the block log to store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log; diff --git a/tests/performance_tests/performance_test.py b/tests/performance_tests/performance_test.py index e20c0ca710..a6cf6a846c 100755 --- a/tests/performance_tests/performance_test.py +++ b/tests/performance_tests/performance_test.py @@ -502,7 +502,7 @@ def main(): chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, - eosVmOcEnable=args.eos_vm_oc_enable, blockLogRetainBlocks=args.block_log_retain_blocks, + blockLogRetainBlocks=args.block_log_retain_blocks, abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) producerPluginArgs = ProducerPluginArgs(disableSubjectiveBilling=args.disable_subjective_billing, lastBlockTimeOffsetUs=args.last_block_time_offset_us, produceTimeOffsetUs=args.produce_time_offset_us, diff --git a/tests/performance_tests/performance_test_basic.py b/tests/performance_tests/performance_test_basic.py index 6ed9eb7777..d002c1500f 100755 --- a/tests/performance_tests/performance_test_basic.py +++ b/tests/performance_tests/performance_test_basic.py @@ -90,7 +90,7 @@ class SpecifiedContract: nodeosVers: str = "" specificExtraNodeosArgs: dict = field(default_factory=dict) _totalNodes: int = 2 - nonprodsEnableEosVmOcEnable: bool = False + nonProdsEosVmOcEnable: bool = False def log_transactions(self, trxDataFile, block): for trx in block['payload']['transactions']: @@ -100,10 +100,12 @@ def log_transactions(self, trxDataFile, block): def __post_init__(self): self._totalNodes = self.pnodes + 1 if self.totalNodes <= self.pnodes else self.totalNodes + nonProdsSpecificNodeosStr = "" if not self.prodsEnableTraceApi: - self.specificExtraNodeosArgs.update({f"{node}" : "--plugin eosio::trace_api_plugin" for node in range(self.pnodes, self._totalNodes)}) - if self.nonprodsEnableEosVmOcEnable: - self.specificExtraNodeosArgs.update({f"{node}" : "--eos-vm-oc-enable" for node in range(self.pnodes, self._totalNodes)}) + nonProdsSpecificNodeosStr += "--plugin eosio::trace_api_plugin " + if self.nonProdsEosVmOcEnable: + nonProdsSpecificNodeosStr += "--eos-vm-oc-enable " + self.specificExtraNodeosArgs.update({f"{node}" : nonProdsSpecificNodeosStr for node in range(self.pnodes, self._totalNodes)}) assert self.nodeosVers != "v1" and self.nodeosVers != "v0", f"nodeos version {Utils.getNodeosVersion().split('.')[0]} is unsupported by performance test" if self.nodeosVers == "v2": self.fetchBlock = lambda node, blockNum: node.processUrllibRequest("chain", "get_block", {"block_num_or_id":blockNum}, silentErrors=False, exitOnError=True) @@ -563,7 +565,7 @@ def createBaseArgumentParser(): ptbBaseParserGroup.add_argument("--contracts-console", help="print contract's output to console", action='store_true') ptbBaseParserGroup.add_argument("--eos-vm-oc-cache-size-mb", type=int, help="Maximum size (in MiB) of the EOS VM OC code cache", default=1024) ptbBaseParserGroup.add_argument("--eos-vm-oc-compile-threads", type=int, help="Number of threads to use for EOS VM OC tier-up", default=1) - ptbBaseParserGroup.add_argument("--eos-vm-oc-enable", help="Enable EOS VM OC tier-up runtime", action='store_true') + ptbBaseParserGroup.add_argument("--non-prods-eos-vm-oc-enable", help="Enable EOS VM OC tier-up runtime on non producer nodes", action='store_true') ptbBaseParserGroup.add_argument("--block-log-retain-blocks", type=int, help="If set to greater than 0, periodically prune the block log to\ store only configured number of most recent blocks. If set to 0, no blocks are be written to the block log;\ block log file is removed after startup.", default=None) @@ -605,7 +607,8 @@ def main(): chainPluginArgs = ChainPluginArgs(signatureCpuBillablePct=args.signature_cpu_billable_pct, chainThreads=args.chain_threads, databaseMapMode=args.database_map_mode, wasmRuntime=args.wasm_runtime, contractsConsole=args.contracts_console, - eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, blockLogRetainBlocks=args.block_log_retain_blocks, + eosVmOcCacheSizeMb=args.eos_vm_oc_cache_size_mb, eosVmOcCompileThreads=args.eos_vm_oc_compile_threads, + blockLogRetainBlocks=args.block_log_retain_blocks, abiSerializerMaxTimeMs=990000, chainStateDbSizeMb=256000) lbto = args.last_block_time_offset_us @@ -630,7 +633,7 @@ def main(): testClusterConfig = PerformanceTestBasic.ClusterConfig(pnodes=args.p, totalNodes=args.n, topo=args.s, genesisPath=args.genesis, prodsEnableTraceApi=args.prods_enable_trace_api, extraNodeosArgs=extraNodeosArgs, specifiedContract=specifiedContract, loggingLevel=args.cluster_log_lvl, - nodeosVers=Utils.getNodeosVersion().split('.')[0], nonprodsEnableEosVmOcEnable=args.eos_vm_oc_enable) + nodeosVers=Utils.getNodeosVersion().split('.')[0], nonProdsEosVmOcEnable=args.non_prods_eos_vm_oc_enable) if args.contracts_console and testClusterConfig.loggingLevel != "debug" and testClusterConfig.loggingLevel != "all": print("Enabling contracts-console will not print anything unless debug level is 'debug' or higher." From 70754472657f8c0b80d6d976534b8b403d652b54 Mon Sep 17 00:00:00 2001 From: Clayton Calabrese Date: Tue, 28 Feb 2023 17:28:03 -0600 Subject: [PATCH 6/6] add fix for duplicate transaction in transaction generators when resigning transactions --- tests/trx_generator/trx_generator.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/trx_generator/trx_generator.cpp b/tests/trx_generator/trx_generator.cpp index 97ec8d948a..1f8a8af891 100644 --- a/tests/trx_generator/trx_generator.cpp +++ b/tests/trx_generator/trx_generator.cpp @@ -55,7 +55,9 @@ namespace eosio::testing { void trx_generator_base::update_resign_transaction(signed_transaction& trx, const fc::crypto::private_key& priv_key, uint64_t& nonce_prefix, uint64_t& nonce, const fc::microseconds& trx_expiration, const chain_id_type& chain_id, const block_id_type& last_irr_block_id) { trx.context_free_actions.clear(); - trx.context_free_actions.emplace_back(action({}, config::null_account_name, name("nonce"), fc::raw::pack(std::to_string(nonce_prefix) + ":" + std::to_string(++nonce) + ":" + fc::time_point::now().time_since_epoch().count()))); + trx.context_free_actions.emplace_back(action({}, config::null_account_name, name("nonce"), + fc::raw::pack(std::to_string(_generator_id) + ":" + std::to_string(nonce_prefix) + ":" + std::to_string(++nonce) + ":" + + fc::time_point::now().time_since_epoch().count()))); set_transaction_headers(trx, last_irr_block_id, trx_expiration); trx.signatures.clear(); trx.sign(priv_key, chain_id);