Skip to content

Commit

Permalink
Merge pull request #1429 from eisenhauer/SstPythonicTesting
Browse files Browse the repository at this point in the history 
Switch staging-common to a Python test script
  • Loading branch information
@eisenhauer
eisenhauer authored May 19, 2019
2 parents bf6f7b0 + e18cb65 commit 83da09c
Show file tree
Hide file tree
Showing 5 changed files with 310 additions and 393 deletions.
42 changes: 27 additions & 15 deletions testing/adios2/engine/staging-common/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,25 +84,42 @@ if(ADIOS2_HAVE_MPI)
endif()

configure_file(
run_staging_test.in
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/run_staging_test
run_test.in
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/run_test.py
@ONLY
)

configure_file(
run_multi_test.in
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/run_multi_test
@ONLY
)

include ( TestSupp.cmake )
include (TestSupp.cmake )

if (ADIOS2_HAVE_MPI)
# we want to know if mpiexec is a shell script or a binary executable
file(READ ${MPIEXEC} MPIFILEDATA LIMIT 1024 HEX)
string(LENGTH ${MPIFILEDATA} DATALENGTH)
math(EXPR last_hex_index "(${DATALENGTH} / 2) - 1")
foreach(hex_index RANGE ${last_hex_index})
math (EXPR str_loc "2*${hex_index}")
string(SUBSTRING "${MPIFILEDATA}" "${str_loc}" "2" char)
#math (EXPR dec_char "0x${char}" )
from_hex(${char} dec_char)
if ((${dec_char} LESS 9) OR (${dec_char} GREATER 126) OR ((${dec_char} GREATER 13) AND (${dec_char} LESS 32)))
# message ("$character ${char} is outside of the printable/character ascii range")
set (MPIEXEC_IS_BINARY TRUE)
break()
endif()
endforeach()
endif()

set (TEST_SET "1x1;NoReaderNoWait;Modes;1x1.Attrs;1x1.Local")
set (FORTRAN_TESTS "")
if(ADIOS2_HAVE_Fortran)
set (FORTRAN_TESTS "FtoC.1x1;CtoF.1x1;FtoF.1x1")
endif()
set (SPECIAL_TESTS "KillReadersSerialized;KillReaders3Max;TimeoutReader;LatestReader;DiscardWriter;PreciousTimestep;PreciousTimestepDiscard")

set (SPECIAL_TESTS "TimeoutReader;LatestReader;DiscardWriter;PreciousTimestep;PreciousTimestepDiscard")
if (MPIEXEC_IS_BINARY)
# run_test.py can only kill readers/writers if mpiexec is not a shell script
list(APPEND SPECIAL_TESTS "KillReadersSerialized;KillReaders3Max;KillWriter_2x2;KillWriterTimeout_2x2")
endif()

set (MPI_TESTS "")
set (MPI_FORTRAN_TESTS "")
Expand Down Expand Up @@ -137,16 +154,11 @@ MutateTestSet( BP_SST_TESTS "BP" "MarshalMethod:BP" "${COMM_MIN_SST_TESTS};${COM
set (SST_TESTS "")
LIST (APPEND SST_TESTS ${FFS_SST_TESTS} ${BP_SST_TESTS})

list( LENGTH SST_TESTS beforelistlen )

# remove Fto anything tests that use FFS because we can't spec it
list(FILTER SST_TESTS EXCLUDE REGEX "Fto.*FFS.*")
# remove Fto anything tests that use CommMin because we can't spec it
list(FILTER SST_TESTS EXCLUDE REGEX "Fto.*CommMin.*")

list( LENGTH SST_TESTS afterlistlen )
message (STATUS "Staging tests list before was ${beforelistlen}, after is ${afterlistlen}")

foreach(test ${SST_TESTS})
add_common_test(${test} SST)
endforeach()
Expand Down
102 changes: 67 additions & 35 deletions testing/adios2/engine/staging-common/TestSupp.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
# add_common_test(1x1 SST) ends up doing:
#
# add_test(NAME "Staging.1x1.SST"
# COMMAND "run_staging_test -e SST -f Staging.1x1.SST -nw 1 -nr 1 -v -p TestCommon")
# COMMAND "run_staging_test -e SST -f Staging.1x1.SST -nw 1 -nr 1")
# set_tests_properties(${testname} PROPERTIES TIMEOUT 30 RUN_SERIAL 1)
#
# RUNNING TESTS WITH DIFFERENT ENGINE PARAMETERS
Expand All @@ -37,6 +37,7 @@
# with different engine parameters, *_CMD strings can also contain the
# string "ENGINE_PARAMS". This string is treated specially by the
# function MutateTestSet(). This function is takes a list of tests

# (I.E. things with _CMD strings defined like above) and creates a new
# set of tests where a specified engine parameter gets added to the in
# the location of the ENGINE_PARAMS string. MutateTestSet takes 4 parameters:
Expand All @@ -58,75 +59,76 @@
#
set (STAGING_COMMON_TEST_SUPP_VERBOSE OFF)

set (1x1_CMD "run_staging_test -nw 1 -nr 1 -v -p TestCommon -arg ENGINE_PARAMS")
set (2x1_CMD "run_staging_test -nw 2 -nr 1 -v -p TestCommon -arg ENGINE_PARAMS")
set (1x2_CMD "run_staging_test -nw 1 -nr 2 -v -p TestCommon -arg ENGINE_PARAMS")
set (3x5_CMD "run_staging_test -nw 3 -nr 5 -v -p TestCommon -arg ENGINE_PARAMS")
set (5x3_CMD "run_staging_test -nw 5 -nr 3 -v -p TestCommon -arg ENGINE_PARAMS")
set (1x1.Local_CMD "run_staging_test -nw 1 -nr 1 -v -w TestCommonWriteLocal -r TestCommonReadLocal -arg ENGINE_PARAMS")
set (2x1.Local_CMD "run_staging_test -nw 2 -nr 1 -v -w TestCommonWriteLocal -r TestCommonReadLocal -arg ENGINE_PARAMS")
set (1x2.Local_CMD "run_staging_test -nw 1 -nr 2 -v -w TestCommonWriteLocal -r TestCommonReadLocal -arg ENGINE_PARAMS")
set (3x5.Local_CMD "run_staging_test -nw 3 -nr 5 -v -w TestCommonWriteLocal -r TestCommonReadLocal -arg ENGINE_PARAMS")
set (5x3.Local_CMD "run_staging_test -nw 5 -nr 3 -v -w TestCommonWriteLocal -r TestCommonReadLocal -arg ENGINE_PARAMS")
set (DelayedReader_3x5_CMD "run_staging_test -rd 5 -nw 3 -nr 5 -p TestCommon -arg ENGINE_PARAMS")
set (FtoC.3x5_CMD "run_staging_test -nw 3 -nr 5 -v -w TestCommonWrite_f -r TestCommonRead -arg ENGINE_PARAMS")
set (FtoF.3x5_CMD "run_staging_test -nw 3 -nr 5 -v -w TestCommonWrite_f -r TestCommonRead_f -arg ENGINE_PARAMS")
set (1x1_CMD "run_test.py -nw 1 -nr 1 --warg=ENGINE_PARAMS")
set (2x1_CMD "run_test.py -nw 2 -nr 1 --warg=ENGINE_PARAMS")
set (1x2_CMD "run_test.py -nw 1 -nr 2 --warg=ENGINE_PARAMS")
set (3x5_CMD "run_test.py -nw 3 -nr 5 --warg=ENGINE_PARAMS")
set (5x3_CMD "run_test.py -nw 5 -nr 3 --warg=ENGINE_PARAMS")
set (1x1.Local_CMD "run_test.py -nw 1 -nr 1 -w TestCommonWriteLocal -r TestCommonReadLocal --warg=ENGINE_PARAMS")
set (2x1.Local_CMD "run_test.py -nw 2 -nr 1 -w TestCommonWriteLocal -r TestCommonReadLocal --warg=ENGINE_PARAMS")
set (1x2.Local_CMD "run_test.py -nw 1 -nr 2 -w TestCommonWriteLocal -r TestCommonReadLocal --warg=ENGINE_PARAMS")
set (3x5.Local_CMD "run_test.py -nw 3 -nr 5 -w TestCommonWriteLocal -r TestCommonReadLocal --warg=ENGINE_PARAMS")
set (5x3.Local_CMD "run_test.py -nw 5 -nr 3 -w TestCommonWriteLocal -r TestCommonReadLocal --warg=ENGINE_PARAMS")
set (DelayedReader_3x5_CMD "run_test.py -rd 5 -nw 3 -nr 5 --warg=ENGINE_PARAMS")
set (FtoC.3x5_CMD "run_test.py -nw 3 -nr 5 -w TestCommonWrite_f -r TestCommonRead --warg=ENGINE_PARAMS")
set (FtoF.3x5_CMD "run_test.py -nw 3 -nr 5 -w TestCommonWrite_f -r TestCommonRead_f --warg=ENGINE_PARAMS")

# NoReaderNoWait runs a writer with the RendezvousReaderCount = 0 and then never spawns a reader. The test should run to termination and execute cleanly
set (NoReaderNoWait_CMD "run_staging_test -nw 1 -nr 0 -v -p TestCommon -arg RendezvousReaderCount:0,QueueLimit:3,QueueFullPolicy:discard,ENGINE_PARAMS")
set (NoReaderNoWait_CMD "run_test.py -nw 1 -nr 0 --warg=RendezvousReaderCount:0,QueueLimit:3,QueueFullPolicy:discard,ENGINE_PARAMS")

# The Modes test checks to see that we can mix and match Put Sync and Deferred modes and still get good data
set (Modes_CMD "run_staging_test -nw 1 -nr 1 -v -w TestCommonWriteModes -r TestCommonRead -arg ENGINE_PARAMS")
set (Modes_CMD "run_test.py -nw 1 -nr 1 -w TestCommonWriteModes -r TestCommonRead --warg=ENGINE_PARAMS")

# 1x1.Attrs tests writing and reading of attributes defined before Open
set (1x1.Attrs_CMD "run_staging_test -nw 1 -nr 1 -v -w TestCommonWriteAttrs -r TestCommonReadAttrs -arg ENGINE_PARAMS")
set (1x1.Attrs_CMD "run_test.py -nw 1 -nr 1 -w TestCommonWriteAttrs -r TestCommonReadAttrs --warg=ENGINE_PARAMS")

# Basic Fortran tests, Fortran to C, C to Fortran and Fortran to Fortran
set (FtoC.1x1_CMD "run_staging_test -nw 1 -nr 1 -v -w TestCommonWrite_f -r TestCommonRead -arg ENGINE_PARAMS")
set (CtoF.1x1_CMD "run_staging_test -nw 1 -nr 1 -v -w TestCommonWrite -r TestCommonRead_f -arg ENGINE_PARAMS")
set (FtoF.1x1_CMD "run_staging_test -nw 1 -nr 1 -v -w TestCommonWrite_f -r TestCommonRead_f -arg ENGINE_PARAMS")
set (FtoC.1x1_CMD "run_test.py -nw 1 -nr 1 -w TestCommonWrite_f -r TestCommonRead --warg=ENGINE_PARAMS")
set (CtoF.1x1_CMD "run_test.py -nw 1 -nr 1 -w TestCommonWrite -r TestCommonRead_f --warg=ENGINE_PARAMS")
set (FtoF.1x1_CMD "run_test.py -nw 1 -nr 1 -w TestCommonWrite_f -r TestCommonRead_f --warg=ENGINE_PARAMS")

# Tests for ZFP compression (where supported by an engine param)
set (ZFPCompression.1x1_CMD "run_staging_test -nw 1 -nr 1 -v -p TestCommon -arg CompressionMethod:zfp,ENGINE_PARAMS" )
set (ZFPCompression.3x5_CMD "run_staging_test -nw 3 -nr 5 -v -p TestCommon -arg CompressionMethod:zfp,ENGINE_PARAMS" )
set (ZFPCompression.1x1_CMD "run_test.py -nw 1 -nr 1 --warg=CompressionMethod:zfp,ENGINE_PARAMS" )
set (ZFPCompression.3x5_CMD "run_test.py -nw 3 -nr 5 --warg=CompressionMethod:zfp,ENGINE_PARAMS" )

# Test if writer will survive readers departing unexpectedly
set (KillReadersSerialized_CMD "run_multi_test -test_protocol kill_readers -verbose -nw 3 -nr 2 -max_readers 1 -warg RendezvousReaderCount:0,ENGINE_PARAMS -rarg --ignore_time_gap")
set (KillReadersSerialized_CMD "run_test.py --test_protocol kill_readers -nw 3 -nr 2 --max_readers 1 --warg=RendezvousReaderCount:0,ENGINE_PARAMS --rarg=--ignore_time_gap")
set (KillReadersSerialized_TIMEOUT "300")
set (KillReadersSerialized_PROPERTIES "RUN_SERIAL;1")
set (KillReaders3Max_CMD "run_multi_test -test_protocol kill_readers -verbose -nw 3 -nr 2 -max_readers 3 -warg RendezvousReaderCount:0,ENGINE_PARAMS -rarg --ignore_time_gap")
set (KillReaders3Max_CMD "run_test.py --test_protocol kill_readers -nw 3 -nr 2 --max_readers 3 --warg=RendezvousReaderCount:0,ENGINE_PARAMS --rarg=--ignore_time_gap")
set (KillReaders3Max_TIMEOUT "300")
set (KillReaders3Max_PROPERTIES "RUN_SERIAL;1")

set (KillWriter_1x2_CMD "run_multi_test -test_protocol kill_writer -verbose -nw 1 -nr 2 -interval 2 -warg RendezvousReaderCount:1,ENGINE_PARAMS -rarg --expect_writer_failure -rarg --num_steps --rarg 1000")
set (KillWriterTimeout_1x2_CMD "run_multi_test -test_protocol kill_writer -verbose -nw 1 -nr 2 -interval 2 -warg RendezvousReaderCount:1,ENGINE_PARAMS -rarg --expect_writer_failure -rarg --num_steps --rarg 1000 --rarg --non_blocking")
set (KillWriter_2x2_CMD "run_test.py --test_protocol kill_writer -nw 2 -nr 2 --interval 2 --warg=RendezvousReaderCount:1,ENGINE_PARAMS --rarg=--expect_writer_failure --rarg=--num_steps --rarg=1000")
set (KillWriterTimeout_2x2_CMD "run_test.py --test_protocol kill_writer -nw 2 -nr 2 --interval 2 --warg=RendezvousReaderCount:1,ENGINE_PARAMS --rarg=--expect_writer_failure --rarg=--num_steps --rarg=1000 --rarg=--non_blocking")

set (PreciousTimestep_CMD "run_test.py --test_protocol kill_readers -nw 3 -nr 2 --max_readers 2 --warg=FirstTimestepPrecious:true,RendezvousReaderCount:0,ENGINE_PARAMS --rarg=--ignore_time_gap --rarg=--precious_first")

set (PreciousTimestep_CMD "run_multi_test -test_protocol kill_readers -verbose -nw 3 -nr 2 -max_readers 2 -warg FirstTimestepPrecious:true,RendezvousReaderCount:0,ENGINE_PARAMS -rarg --ignore_time_gap -rarg --precious_first")
set (PreciousTimestep_TIMEOUT "300")
set (PreciousTimestep_PROPERTIES "RUN_SERIAL;1")

set (PreciousTimestepDiscard_CMD "run_multi_test -test_protocol kill_readers -verbose -nw 3 -nr 2 -max_readers 2 -warg FirstTimestepPrecious:true,RendezvousReaderCount:0,QueueLimit:3,QueueFullPolicy:discard,ENGINE_PARAMS -rarg --ignore_time_gap -rarg --precious_first -rarg --discard -warg --ms_delay -warg 500")
set (PreciousTimestepDiscard_CMD "run_test.py --test_protocol kill_readers -nw 3 -nr 2 --max_readers 2 --warg=FirstTimestepPrecious:true,RendezvousReaderCount:0,QueueLimit:3,QueueFullPolicy:discard,ENGINE_PARAMS --rarg=--ignore_time_gap --rarg=--precious_first --rarg=--discard --warg=--ms_delay --warg=500")
set (PreciousTimestepDiscard_TIMEOUT "300")
set (PreciousTimestepDiscard_PROPERTIES "RUN_SERIAL;1")

# Readers using BeginStep with timeout. Here we run the writer with a longer delay to make the reader timeout
set (TimeoutReader_CMD "run_multi_test -test_protocol one_to_one -verbose -nw 1 -nr 1 -rarg --non_blocking -warg --ms_delay -warg 2000 -warg --engine_params -warg ENGINE_PARAMS")
set (TimeoutReader_CMD "run_test.py --test_protocol one_client -nw 1 -nr 1 --rarg=--non_blocking --warg=--ms_delay --warg=2000 --warg=--engine_params --warg=ENGINE_PARAMS")
set (TimeoutReader_TIMEOUT "60")
set (TimeoutReader_PROPERTIES "RUN_SERIAL;1")

# Readers using LatestAvailable Writer runs faster than reader, so we expect misses
set (LatestReader_CMD "run_multi_test -test_protocol one_to_one -verbose -nw 1 -nr 1 -warg --ms_delay -warg 250 -warg --engine_params -warg ENGINE_PARAMS -rarg --latest -rarg --long_first_delay")
set (LatestReader_CMD "run_test.py --test_protocol one_client -nw 1 -nr 1 --warg=--ms_delay --warg=250 --warg=--engine_params --warg=ENGINE_PARAMS --rarg=--latest --rarg=--long_first_delay")
set (LatestReader_PROPERTIES "RUN_SERIAL;1")

# A faster writer and a queue policy that will cause timesteps to be discarded
set (DiscardWriter_CMD "run_multi_test -test_protocol one_to_one -verbose -nw 1 -nr 1 -warg --engine_params -warg QueueLimit:1,QueueFullPolicy:discard,ENGINE_PARAMS -warg --ms_delay -warg 250 -rarg --discard")
set (DiscardWriter_CMD "run_test.py --test_protocol one_client -nw 1 -nr 1 --warg=--engine_params --warg=QueueLimit:1,QueueFullPolicy:discard,ENGINE_PARAMS --warg=--ms_delay --warg=250 --rarg=--discard")

function(remove_engine_params_placeholder dst_str src_str )
string(REGEX REPLACE "([^ ]*),ENGINE_PARAMS" "\\1" src_str "${src_str}")
if ("${src_str}" MATCHES "ENGINE_PARAMS")
# empty engine params remains
string(REGEX REPLACE "-warg *--engine_params *-warg *ENGINE_PARAMS" "" src_str "${src_str}")
string(REGEX REPLACE "-warg *--engine_params *-rarg *ENGINE_PARAMS" "" src_str "${src_str}")
string(REGEX REPLACE "--warg=--engine_params --warg=ENGINE_PARAMS" "" src_str "${src_str}")
string(REGEX REPLACE "--rarg=--engine_params --rarg=ENGINE_PARAMS" "" src_str "${src_str}")
string(REGEX REPLACE "-arg *ENGINE_PARAMS" "" src_str "${src_str}")
endif()
set(${dst_str} ${src_str} PARENT_SCOPE)
Expand Down Expand Up @@ -170,14 +172,13 @@ endfunction()

function(add_common_test basename engine)
set(testname "Staging.${basename}.${engine}")
set(filename "Staging.${basename}.${engine}")
if ("${${basename}_CMD}" STREQUAL "")
message(SEND_ERROR "Staging-Common test ${basename} has no defined ${basename}_CMD")
endif()
string (CONCAT command "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/" ${${basename}_CMD})
remove_engine_params_placeholder(command "${command}")
separate_arguments(command)
list(INSERT command 1 "-e" "${engine}" "-f" "${filename}")
list(INSERT command 1 "${engine}" "${testname}")
add_test(
NAME ${testname}
COMMAND ${command})
Expand All @@ -192,3 +193,34 @@ function(add_common_test basename engine)
set_tests_properties(${testname} PROPERTIES TIMEOUT ${timeout} ${${basename}_PROPERTIES} )
endfunction()

function(from_hex HEX DEC)
string(TOUPPER "${HEX}" HEX)
set(_res 0)
string(LENGTH "${HEX}" _strlen)

while(_strlen GREATER 0)
math(EXPR _res "${_res} * 16")
string(SUBSTRING "${HEX}" 0 1 NIBBLE)
string(SUBSTRING "${HEX}" 1 -1 HEX)
if(NIBBLE STREQUAL "A")
math(EXPR _res "${_res} + 10")
elseif(NIBBLE STREQUAL "B")
math(EXPR _res "${_res} + 11")
elseif(NIBBLE STREQUAL "C")
math(EXPR _res "${_res} + 12")
elseif(NIBBLE STREQUAL "D")
math(EXPR _res "${_res} + 13")
elseif(NIBBLE STREQUAL "E")
math(EXPR _res "${_res} + 14")
elseif(NIBBLE STREQUAL "F")
math(EXPR _res "${_res} + 15")
else()
math(EXPR _res "${_res} + ${NIBBLE}")
endif()

string(LENGTH "${HEX}" _strlen)
endwhile()

set(${DEC} ${_res} PARENT_SCOPE)
endfunction()

Loading

0 comments on commit 83da09c

Please sign in to comment.