From beb86bd997632a354be00abe757ad8cecb483f53 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 20:59:38 -0500 Subject: [PATCH] WX-1351 Speed up `Centaur Horicromtal PapiV2 Beta` (#7329) --- core/src/main/resources/reference.conf | 3 +- src/ci/bin/test.inc.sh | 39 +++++++++++++++++++ .../bin/testCentaurHoricromtalPapiV2beta.sh | 4 +- src/ci/bin/testCentaurPapiV2beta.sh | 2 + .../papi_v2beta_horicromtal_application.conf | 6 +-- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 6ec05cf6025..3fa11b458ff 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -118,7 +118,8 @@ system { max-concurrent-workflows = 5000 # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist - max-workflow-launch-count = 50 + # Deviating from 1 is not recommended for multi-runner setups due to possible deadlocks. [BW-962] + max-workflow-launch-count = 1 # Workflows will be grouped by the value of the specified field in their workflow options. # diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index eef3a9fab6b..7da8290b9a1 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1607,6 +1607,45 @@ cromwell::build::generate_code_coverage() { fi } +cromwell::build::print_workflow_statistics() { + echo "Total workflows" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT COUNT(*) as total_workflows_run FROM WORKFLOW_METADATA_SUMMARY_ENTRY;" + + echo "Late starters" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT WORKFlOW_NAME as name, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, + START_TIMESTAMP as START, + END_TIMESTAMP as end + FROM WORKFLOW_METADATA_SUMMARY_ENTRY + WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL # exclude subworkflows + ORDER BY START_TIMESTAMP DESC + LIMIT 20;" + + echo "Late finishers" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT WORKFlOW_NAME as name, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, + START_TIMESTAMP as start, + END_TIMESTAMP as END + FROM WORKFLOW_METADATA_SUMMARY_ENTRY + WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL + ORDER BY END_TIMESTAMP DESC + LIMIT 20;" + + echo "Long duration" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT WORKFlOW_NAME as name, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as RUNTIME_MINUTES, + START_TIMESTAMP as start, + END_TIMESTAMP as end + FROM WORKFLOW_METADATA_SUMMARY_ENTRY + WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL + ORDER BY RUNTIME_MINUTES DESC + LIMIT 20;" +} + cromwell::build::exec_retry_function() { local retried_function local retry_count diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index 9dba2f6f8ce..36b215a2f52 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -19,7 +19,7 @@ cromwell::build::assemble_jars cromwell::build::build_cromwell_docker cromwell::build::run_centaur \ - -p 100 \ + -p 500 \ -e localdockertest \ -e relative_output_paths \ -e relative_output_paths_colliding \ @@ -27,3 +27,5 @@ cromwell::build::run_centaur \ -e papi_v2alpha1_gcsa \ cromwell::build::generate_code_coverage + +cromwell::build::print_workflow_statistics diff --git a/src/ci/bin/testCentaurPapiV2beta.sh b/src/ci/bin/testCentaurPapiV2beta.sh index b3a1ef2a61e..43d5a7d62aa 100755 --- a/src/ci/bin/testCentaurPapiV2beta.sh +++ b/src/ci/bin/testCentaurPapiV2beta.sh @@ -25,3 +25,5 @@ cromwell::build::run_centaur \ -e papi_v2alpha1_gcsa \ cromwell::build::generate_code_coverage + +cromwell::build::print_workflow_statistics diff --git a/src/ci/resources/papi_v2beta_horicromtal_application.conf b/src/ci/resources/papi_v2beta_horicromtal_application.conf index c6bc9176c16..f0f2c45f381 100644 --- a/src/ci/resources/papi_v2beta_horicromtal_application.conf +++ b/src/ci/resources/papi_v2beta_horicromtal_application.conf @@ -1,8 +1,8 @@ include "papi_v2beta_application.conf" -system.max-workflow-launch-count=1 -system.new-workflow-poll-rate=10 -system.max-concurrent-workflows=30 +system.max-workflow-launch-count=10 +system.new-workflow-poll-rate=1 +system.max-concurrent-workflows=500 system.cromwell_id_random_suffix=false # Turn off token logging to reduce log volume