Failed the sample tests when it should (#417)

* failed the sample tests when it should * print log before failure
kubeflow · Nov 30, 2018 · 548a2d6 · 548a2d6
1 parent 537357a
commit 548a2d6
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 15 deletions.
diff --git a/test/sample-test/run_basic_test.py b/test/sample-test/run_basic_test.py
@@ -64,7 +64,8 @@ def main():
   utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated')
   if not os.path.exists(args.input):
     utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
+    print('Error: job not found.')
+    exit(1)
 
   ###### Create Experiment ######
   experiment_name = args.testname + ' sample experiment'
@@ -87,9 +88,6 @@ def main():
   end_time = datetime.now()
   elapsed_time = (end_time - start_time).seconds
   utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time)
-  if not succ:
-    utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
 
   ###### Output Argo Log for Debugging ######
   workflow_json = client._get_workflow_json(run_id)
@@ -98,6 +96,10 @@ def main():
   print("=========Argo Workflow Log=========")
   print(argo_log)
 
+  if not succ:
+    utils.write_junit_xml(test_name, args.result, test_cases)
+    exit(1)
+
   ###### Delete Job ######
   #TODO: add deletion when the backend API offers the interface.
 

diff --git a/test/sample-test/run_kubeflow_test.py b/test/sample-test/run_kubeflow_test.py
@@ -61,7 +61,8 @@ def main():
   utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated')
   if not os.path.exists(args.input):
     utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
+    print('Error: job not found.')
+    exit(1)
 
   ###### Create Experiment ######
   experiment_name = 'kubeflow sample experiment'
@@ -88,9 +89,6 @@ def main():
   end_time = datetime.now()
   elapsed_time = (end_time - start_time).seconds
   utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time)
-  if not succ:
-    utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
 
   ###### Output Argo Log for Debugging ######
   workflow_json = client._get_workflow_json(run_id)
@@ -99,6 +97,10 @@ def main():
   print("=========Argo Workflow Log=========")
   print(argo_log)
 
+  if not succ:
+    utils.write_junit_xml(test_name, args.result, test_cases)
+    exit(1)
+
   ###### Validate the results ######
   #   confusion matrix should show three columns for the flower data
   #     target, predicted, count

diff --git a/test/sample-test/run_tfx_test.py b/test/sample-test/run_tfx_test.py
@@ -60,7 +60,8 @@ def main():
   utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated')
   if not os.path.exists(args.input):
     utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
+    print('Error: job not found.')
+    exit(1)
 
   ###### Create Experiment ######
   experiment_name = 'TFX sample experiment'
@@ -81,17 +82,13 @@ def main():
   run_id = response.id
   utils.add_junit_test(test_cases, 'create pipeline run', True)
 
-
   ###### Monitor Job ######
   start_time = datetime.now()
   response = client.wait_for_run_completion(run_id, 1200)
   succ = (response.run.status.lower()=='succeeded')
   end_time = datetime.now()
   elapsed_time = (end_time - start_time).seconds
   utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time)
-  if not succ:
-    utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
 
   ###### Output Argo Log for Debugging ######
   workflow_json = client._get_workflow_json(run_id)
@@ -100,6 +97,10 @@ def main():
   print("=========Argo Workflow Log=========")
   print(argo_log)
 
+  if not succ:
+    utils.write_junit_xml(test_name, args.result, test_cases)
+    exit(1)
+
   ###### Validate the results ######
   #TODO: enable after launch
   #   model analysis html is validated

diff --git a/test/sample-test/run_xgboost_test.py b/test/sample-test/run_xgboost_test.py
@@ -61,7 +61,8 @@ def main():
   utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated')
   if not os.path.exists(args.input):
     utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
+    print('Error: job not found.')
+    exit(1)
 
   ###### Create Experiment ######
   experiment_name = 'xgboost sample experiment'
@@ -100,7 +101,7 @@ def main():
   ###### If the job fails, skip the result validation ######
   if not succ:
     utils.write_junit_xml(test_name, args.result, test_cases)
-    exit()
+    exit(1)
 
   ###### Validate the results ######
   #   confusion matrix should show three columns for the flower data