add sequential sample test

test/sample-test/run_sequential_test.py

@@ -0,0 +1,101 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import os
+from datetime import datetime
+from kfp import Client
+import utils
+
+###### Input/Output Instruction ######
+# input: yaml
+# output: local file path
+
+
+# Parsing the input arguments
+def parse_arguments():
+  """Parse command line arguments."""
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--input',
+                      type=str,
+                      required=True,
+                      help='The path of a pipeline package that will be submitted.')
+  parser.add_argument('--result',
+                      type=str,
+                      required=True,
+                      help='The path of the test result that will be exported.')
+  parser.add_argument('--output',
+                      type=str,
+                      required=True,
+                      help='The path of the test output')
+  args = parser.parse_args()
+  return args
+
+def main():
+  args = parse_arguments()
+  test_cases = []
+  test_name = 'Sequential Sample Test'
+
+  ###### Initialization ######
+  client = Client()
+
+  ###### Check Input File ######
+  utils.add_junit_test(test_cases, 'input generated yaml file', os.path.exists(args.input), 'yaml file is not generated')
+  if not os.path.exists(args.input):
+    utils.write_junit_xml(test_name, args.result, test_cases)
+    exit()
+
+  ###### Create Experiment ######
+  experiment_name = 'Sequential sample experiment'
+  response = client.create_experiment(experiment_name)
+  experiment_id = response.id
+  utils.add_junit_test(test_cases, 'create experiment', True)
+
+  ###### Create Job ######
+  job_name = 'Sequential_sample'
+  params = {}
+  response = client.run_pipeline(experiment_id, job_name, args.input, params)
+  run_id = response.id
+  utils.add_junit_test(test_cases, 'create pipeline run', True)
+
+
+  ###### Monitor Job ######
+  start_time = datetime.now()
+  response = client.wait_for_run_completion(run_id, 1200)
+  succ = (response.run.status.lower()=='succeeded')
+  end_time = datetime.now()
+  elapsed_time = (end_time - start_time).seconds
+  utils.add_junit_test(test_cases, 'job completion', succ, 'waiting for job completion failure', elapsed_time)
+  if not succ:
+    utils.write_junit_xml(test_name, args.result, test_cases)
+    exit()
+
+  ###### Output Argo Log for Debugging ######
+  workflow_json = client._get_workflow_json(run_id)
+  workflow_id = workflow_json['metadata']['name']
+  #TODO: remove the namespace dependency or make is configurable.
+  argo_log, _ = utils.run_bash_command('argo logs -n kubeflow -w {}'.format(workflow_id))
+  print("=========Argo Workflow Log=========")
+  print(argo_log)
+
+  ###### Delete Job ######
+  #TODO: add deletion when the backend API offers the interface.
+
+  ###### Write out the test result in junit xml ######
+  utils.write_junit_xml(test_name, args.result, test_cases)
+
+if __name__ == "__main__":
+  main()

test/sample-test/run_test.sh

@@ -157,4 +157,14 @@ elif [ "$TEST_NAME" == "tfx" ]; then
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_TFX_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_TFX_TEST_RESULT}
+elif [ "$TEST_NAME" == "sequential" ]; then
+  SAMPLE_SEQUENTIAL_TEST_RESULT=junit_SampleSequentialOutput.xml
+  SAMPLE_SEQUENTIAL_TEST_OUTPUT=${RESULTS_GCS_DIR}
+
+  # Compile samples
+  cd ${BASE_DIR}/samples/basic
+  dsl-compile --py sequential.py --output sequential.tar.gz
+
+  cd /
+  python3 run_sequential_test.py --input ${BASE_DIR}/samples/basic/sequential.tar.gz --result SAMPLE_SEQUENTIAL_TEST_RESULT --output SAMPLE_SEQUENTIAL_TEST_OUTPUT
 fi

test/sample_test.yaml

@@ -227,6 +227,32 @@ spec:
             value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.sample-tests-image-suffix}}"
           - name: test-name
             value: "tfx"
+      - name: run-sequential-tests
+        template: run-sample-tests
+        arguments:
+          parameters:
+          - name: commit-sha
+            value: "{{inputs.parameters.commit-sha}}"
+          - name: test-results-gcs-dir
+            value: "{{inputs.parameters.test-results-gcs-dir}}"
+          - name: dataflow-tft-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.dataflow-tft-image-suffix}}"
+          - name: dataflow-predict-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.dataflow-predict-image-suffix}}"
+          - name: dataflow-tfma-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.dataflow-tfma-image-suffix}}"
+          - name: dataflow-tfdv-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.dataflow-tfdv-image-suffix}}"
+          - name: kubeflow-dnntrainer-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.kubeflow-dnntrainer-image-suffix}}"
+          - name: kubeflow-deployer-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.kubeflow-deployer-image-suffix}}"
+          - name: local-confusionmatrix-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.local-confusionmatrix-image-suffix}}"
+          - name: sample-tests-image
+            value: "gcr.io/{{steps.get-project.outputs.result}}/{{inputs.parameters.commit-sha}}/{{inputs.parameters.sample-tests-image-suffix}}"
+          - name: test-name
+            value: "sequential"
 
   - name: get-project
     script: