Fix flaky basic sample test (#134)

* fix typo * typo fix * start debug mode * fix typos * make the timeout configurable * disable debug mode * update variable name * update timeout to timeout_seconds
kubeflow · Nov 7, 2018 · a540cf7 · a540cf7
1 parent f15dc1e
commit a540cf7
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 7 deletions.
diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh
@@ -23,12 +23,14 @@ usage()
     [--test_result_bucket   the gcs bucket that argo workflow store the result to. Default is ml-pipeline-test
     [--test_result_folder   the gcs folder that argo workflow store the result to. Always a relative directory to gs://<gs_bucket>/[PULL_SHA]]
     [--cluster-type         the type of cluster to use for the tests. One of: create-gke,none. Default is create-gke ]
+    [--timeout              timeout of the tests in seconds. Default is 1800 seconds. ]
     [-h help]"
 }
 
 TEST_RESULT_BUCKET=ml-pipeline-test
 GCR_IMAGE_BASE_DIR=gcr.io/ml-pipeline-test/${PULL_PULL_SHA}
 CLUSTER_TYPE=create-gke
+TIMEOUT_SECONDS=1800
 
 while [ "$1" != "" ]; do
     case $1 in
@@ -44,6 +46,9 @@ while [ "$1" != "" ]; do
              --cluster-type )         shift
                                       CLUSTER_TYPE=$1
                                       ;;
+             --timeout )              shift
+                                      TIMEOUT_SECONDS=$1
+                                      ;;
              -h | --help )            usage
                                       exit
                                       ;;
@@ -54,11 +59,11 @@ while [ "$1" != "" ]; do
 done
 
 ZONE=us-west1-a
-PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=90
 TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${PULL_PULL_SHA}/${TEST_RESULT_FOLDER}
 ARTIFACT_DIR=$WORKSPACE/_artifacts
 WORKFLOW_COMPLETE_KEYWORD="completed=true"
 WORKFLOW_FAILED_KEYWORD="phase=Failed"
+PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
 
 echo "presubmit test starts"
 

diff --git a/test/sample-test/run_test.sh b/test/sample-test/run_test.sh
@@ -194,7 +194,7 @@ elif [ "$TEST_NAME" == "sequential" ]; then
   dsl-compile --py sequential.py --output sequential.tar.gz
 
   cd /
-  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/sequential.tar.gz --result SAMPLE_SEQUENTIAL_TEST_RESULT --output SAMPLE_SEQUENTIAL_TEST_OUTPUT  --testname sequential
+  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/sequential.tar.gz --result $SAMPLE_SEQUENTIAL_TEST_RESULT --output $SAMPLE_SEQUENTIAL_TEST_OUTPUT  --testname sequential
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_SEQUENTIAL_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_SEQUENTIAL_TEST_RESULT}
@@ -207,7 +207,7 @@ elif [ "$TEST_NAME" == "condition" ]; then
   dsl-compile --py condition.py --output condition.tar.gz
 
   cd /
-  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/condition.tar.gz --result SAMPLE_CONDITION_TEST_RESULT --output SAMPLE_CONDITION_TEST_OUTPUT --testname conditio
+  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/condition.tar.gz --result $SAMPLE_CONDITION_TEST_RESULT --output $SAMPLE_CONDITION_TEST_OUTPUT --testname conditio
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_CONDITION_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_CONDITION_TEST_RESULT}
@@ -220,7 +220,7 @@ elif [ "$TEST_NAME" == "exithandler" ]; then
   dsl-compile --py exit_handler.py --output exit_handler.tar.gz
 
   cd /
-  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/exit_handler.tar.gz --result SAMPLE_EXIT_HANDLER_TEST_RESULT --output SAMPLE_EXIT_HANDLER_TEST_OUTPUT --testname exithandler
+  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/exit_handler.tar.gz --result $SAMPLE_EXIT_HANDLER_TEST_RESULT --output $SAMPLE_EXIT_HANDLER_TEST_OUTPUT --testname exithandler
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_EXIT_HANDLER_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_EXIT_HANDLER_TEST_RESULT}
@@ -233,7 +233,7 @@ elif [ "$TEST_NAME" == "immediatevalue" ]; then
   dsl-compile --py immediate_value.py --output immediate_value.tar.gz
 
   cd /
-  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/immediate_value.tar.gz --result SAMPLE_IMMEDIATE_VALUE_TEST_RESULT --output SAMPLE_IMMEDIATE_VALUE_TEST_OUTPUT --testname immediatevalue
+  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/immediate_value.tar.gz --result $SAMPLE_IMMEDIATE_VALUE_TEST_RESULT --output $SAMPLE_IMMEDIATE_VALUE_TEST_OUTPUT --testname immediatevalue
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_IMMEDIATE_VALUE_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_IMMEDIATE_VALUE_TEST_RESULT}
@@ -246,7 +246,7 @@ elif [ "$TEST_NAME" == "paralleljoin" ]; then
   dsl-compile --py parallel_join.py --output parallel_join.tar.gz
 
   cd /
-  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/parallel_join.tar.gz --result SAMPLE_PARALLEL_JOIN_TEST_RESULT --output SAMPLE_PARALLEL_JOIN_TEST_OUTPUT --testname paralleljoin
+  python3 run_basic_test.py --input ${BASE_DIR}/samples/basic/parallel_join.tar.gz --result $SAMPLE_PARALLEL_JOIN_TEST_RESULT --output $SAMPLE_PARALLEL_JOIN_TEST_OUTPUT --testname paralleljoin
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_PARALLEL_JOIN_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_PARALLEL_JOIN_TEST_RESULT}

diff --git a/test/sample_test.yaml b/test/sample_test.yaml
@@ -477,7 +477,7 @@ spec:
           - name: local-roc-image
             value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.local-roc-image-suffix}}"
           - name: sample-tests-image
-            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.sample-tests-image-suffix}}"}"
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.sample-tests-image-suffix}}"
           - name: test-name
             value: "exithandler"
       - name: run-sequential-tests
@@ -567,6 +567,7 @@ spec:
           - name: commit-sha
             value: "{{inputs.parameters.commit-sha}}"
           - name: test-results-gcs-dir
+            value: "{{inputs.parameters.test-results-gcs-dir}}"
           - name: dataflow-tft-image
             value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.dataflow-tft-image-suffix}}"
           - name: dataflow-predict-image