Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create performance harness module #1535

Merged
merged 6 commits into from
Aug 28, 2023
Merged

Create performance harness module #1535

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
43a5c1a
Rename performance_tests to PerformanceHarness.
oschwaldp-oci Aug 18, 2023
c6f9387
Refactor PerformanceHarness into python module.
oschwaldp-oci Aug 18, 2023
8da051a
Remove commented out import.
oschwaldp-oci Aug 18, 2023
938fefe
Merge branch 'main' into create-performance-harness-module
oschwaldp-oci Aug 18, 2023
874e816
Couple minor verbiage tweaks and clarifications.
oschwaldp-oci Aug 28, 2023
76c8abb
Merge branch 'main' into create-performance-harness-module
oschwaldp-oci Aug 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/performance_harness_run.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
- name: Setup Input Params
id: overrides
run: |
echo test-params=testBpOpMode >> $GITHUB_OUTPUT
echo test-params=findMax testBpOpMode >> $GITHUB_OUTPUT

if [[ "${{inputs.override-test-params}}" != "" ]]; then
echo test-params=${{inputs.override-test-params}} >> $GITHUB_OUTPUT
Expand Down Expand Up @@ -103,18 +103,18 @@ jobs:
run: |
zstdcat build.tar.zst | tar x
cd build
./tests/performance_tests/performance_test.py ${{needs.v.outputs.test-params}}
./tests/PerformanceHarnessScenarioRunner.py ${{needs.v.outputs.test-params}}
- name: Prepare results
id: prep-results
run: |
tar -pc build/performance_test | zstd --long -T0 -9 > performance_test_logs.tar.zst
tar -pc build/PerformanceHarnessScenarioRunnerLogs | zstd --long -T0 -9 > PerformanceHarnessScenarioRunnerLogs.tar.zst
- name: Upload results
uses: AntelopeIO/upload-artifact-large-chunks-action@v1
with:
name: performance-test-results
path: performance_test_logs.tar.zst
path: PerformanceHarnessScenarioRunnerLogs.tar.zst
- name: Upload report
uses: actions/upload-artifact@v3
with:
name: performance-test-report
path: ./build/performance_test/**/report.json
path: ./build/PerformanceHarnessScenarioRunnerLogs/**/report.json
26 changes: 25 additions & 1 deletion tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test.py ${CMAKE_CURRE
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test_shape.json COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gelf_test.py ${CMAKE_CURRENT_BINARY_DIR}/gelf_test.py COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/split_blocklog_replay_test.py ${CMAKE_CURRENT_BINARY_DIR}/split_blocklog_replay_test.py COPYONLY)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/PerformanceHarnessScenarioRunner.py ${CMAKE_CURRENT_BINARY_DIR}/PerformanceHarnessScenarioRunner.py COPYONLY)

if(DEFINED ENV{GITHUB_ACTIONS})
set(UNSHARE "--unshared")
Expand Down Expand Up @@ -102,7 +103,7 @@ endif()

add_subdirectory( TestHarness )
add_subdirectory( trx_generator )
add_subdirectory( performance_tests )
add_subdirectory( PerformanceHarness )

find_package(Threads)
add_executable(ship_client ship_client.cpp)
Expand Down Expand Up @@ -271,6 +272,29 @@ set_property(TEST auto_bp_peering_test PROPERTY LABELS long_running_tests)
add_test(NAME gelf_test COMMAND tests/gelf_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
set_property(TEST gelf_test PROPERTY LABELS nonparallelizable_tests)

add_test(NAME performance_test_bp COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_api COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 --calc-chain-threads lmax overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testApiOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --api-nodes-read-only-threads 2 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_cpu_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py findMax testBpOpMode --max-tps-to-test 50 --test-iteration-min-step 10 --test-iteration-duration-sec 10 --final-iterations-duration-sec 10 overrideBasicTestConfig -v --tps-limit-per-generator 25 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_p2p COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_http COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --target-tps 10 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_transfer_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --user-trx-data-file tests/PerformanceHarness/userTrxDataTransfer.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_new_acct_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --user-trx-data-file tests/PerformanceHarness/userTrxDataNewAccount.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_cpu_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "c" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/cpuTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_ram_trx_spec COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --producer-nodes 1 --validation-nodes 1 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "r" --abi-file eosmechanics.abi --wasm-file eosmechanics.wasm --contract-dir unittests/contracts/eosio.mechanics --user-trx-data-file tests/PerformanceHarness/ramTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
add_test(NAME performance_test_basic_read_only_trxs COMMAND tests/PerformanceHarnessScenarioRunner.py singleTest -v --endpoint-mode http --producer-nodes 1 --validation-nodes 1 --api-nodes 1 --api-nodes-read-only-threads 2 --target-tps 20 --tps-limit-per-generator 10 --test-duration-sec 5 --chain-state-db-size-mb 200 --account-name "payloadless" --abi-file payloadless.abi --wasm-file payloadless.wasm --contract-dir unittests/test-contracts/payloadless --user-trx-data-file tests/PerformanceHarness/readOnlyTrxData.json ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
set_property(TEST performance_test_bp PROPERTY LABELS long_running_tests)
set_property(TEST performance_test_api PROPERTY LABELS long_running_tests)
set_property(TEST performance_test_read_only_trxs PROPERTY LABELS long_running_tests)
set_property(TEST performance_test_cpu_trx_spec PROPERTY LABELS long_running_tests)
set_property(TEST performance_test_basic_p2p PROPERTY LABELS nonparallelizable_tests)
set_property(TEST performance_test_basic_http PROPERTY LABELS nonparallelizable_tests)
set_property(TEST performance_test_basic_transfer_trx_spec PROPERTY LABELS nonparallelizable_tests)
set_property(TEST performance_test_basic_new_acct_trx_spec PROPERTY LABELS nonparallelizable_tests)
set_property(TEST performance_test_basic_cpu_trx_spec PROPERTY LABELS nonparallelizable_tests)
set_property(TEST performance_test_basic_ram_trx_spec PROPERTY LABELS nonparallelizable_tests)
set_property(TEST performance_test_basic_read_only_trxs PROPERTY LABELS nonparallelizable_tests)

if(ENABLE_COVERAGE_TESTING)

set(Coverage_NAME ${PROJECT_NAME}_coverage)
Expand Down
11 changes: 11 additions & 0 deletions tests/PerformanceHarness/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
configure_file(performance_test_basic.py . COPYONLY)
configure_file(performance_test.py . COPYONLY)
configure_file(log_reader.py . COPYONLY)
configure_file(genesis.json . COPYONLY)
configure_file(cpuTrxData.json . COPYONLY)
configure_file(ramTrxData.json . COPYONLY)
configure_file(readOnlyTrxData.json . COPYONLY)
configure_file(userTrxDataTransfer.json . COPYONLY)
configure_file(userTrxDataNewAccount.json . COPYONLY)

add_subdirectory( NodeosPluginArgs )
0 ..._tests/NodeosPluginArgs/BasePluginArgs.py → ...arness/NodeosPluginArgs/BasePluginArgs.py
View file
Open in desktop
File renamed without changes.
0 ...nce_tests/NodeosPluginArgs/CMakeLists.txt → ...ceHarness/NodeosPluginArgs/CMakeLists.txt
View file
Open in desktop
File renamed without changes.
0 ...rmance_tests/NodeosPluginArgs/__init__.py → ...manceHarness/NodeosPluginArgs/__init__.py
View file
Open in desktop
File renamed without changes.
0 ...enerate_nodeos_plugin_args_class_files.py → ...enerate_nodeos_plugin_args_class_files.py
View file
Open in desktop
File renamed without changes.
Loading