From 4f3fdf979615ec7db4db2315726873001e79cb0e Mon Sep 17 00:00:00 2001
From: Ning Gao <ngao@google.com>
Date: Tue, 30 Apr 2019 11:47:32 -0700
Subject: [PATCH 1/2] update the image in the samples to use the new component
 images

---
 ...stsubmit-tests-with-pipeline-deployment.sh |  2 +-
 test/sample-test/run_test.sh                  | 36 +++++++++++++------
 2 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/test/postsubmit-tests-with-pipeline-deployment.sh b/test/postsubmit-tests-with-pipeline-deployment.sh
index a791cf10b8d..35d6d4fbede 100755
--- a/test/postsubmit-tests-with-pipeline-deployment.sh
+++ b/test/postsubmit-tests-with-pipeline-deployment.sh
@@ -126,7 +126,7 @@ echo "submitting argo workflow for commit ${PULL_BASE_SHA}..."
 ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
 -p image-build-context-gcs-uri="$remote_code_archive_uri" \
 -p commit-sha="${PULL_BASE_SHA}" \
--p component-image-prefix="${GCR_IMAGE_BASE_DIR}" \
+-p component-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
 -p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \
 -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
 -p cluster-type="${CLUSTER_TYPE}" \
diff --git a/test/sample-test/run_test.sh b/test/sample-test/run_test.sh
index aea7b2f18ce..dc0348bd0f3 100755
--- a/test/sample-test/run_test.sh
+++ b/test/sample-test/run_test.sh
@@ -168,14 +168,24 @@ elif [ "$TEST_NAME" == "tfx" ]; then
   cd ${BASE_DIR}/samples/tfx
 
   if [ -n "${DATAFLOW_TFT_IMAGE}" ];then
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFT_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_PREDICT_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFDV_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFMA_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DNNTRAINER_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DEPLOYER_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" taxi-cab-classification-pipeline.py
+    # Update the image tag in the yaml.
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFT_IMAGE}|g" ../../components/dataflow/tft/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_PREDICT_IMAGE}|g" ../../components/dataflow/predict/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFDV_IMAGE}|g" ../../components/dataflow/tfdv/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFMA_IMAGE}|g" ../../components/dataflow/tfma/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DNNTRAINER_IMAGE}|g" ../../components/kubeflow/dnntrainer/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DEPLOYER_IMAGE}|g" ../../components/kubeflow/deployer/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" ../../components/local/confusion_matrix/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" ../../components/local/roc/component.yaml
+    # Update the yaml reference in the sample
+    sed -i -e "s|dataflow_tf_data_validation_op  = components.load_component_from_url.*|dataflow_tf_data_validation_op = components.load_component_from_file('../../components/dataflow/tfdv/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|dataflow_tf_transform_op        = components.load_component_from_url.*|dataflow_tf_transform_op = components.load_component_from_file('../../components/dataflow/tft/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|tf_train_op                     = components.load_component_from_url.*|tf_train_op = components.load_component_from_file('../../components/kubeflow/dnntrainer/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|dataflow_tf_model_analyze_op    = components.load_component_from_url.*|dataflow_tf_model_analyze_op = components.load_component_from_file('../../components/dataflow/tfma/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|dataflow_tf_predict_op          = components.load_component_from_url.*|dataflow_tf_predict_op = components.load_component_from_file('../../components/dataflow/predict/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|confusion_matrix_op             = components.load_component_from_url.*|confusion_matrix_op = components.load_component_from_file('../../components/local/confusion_matrix/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|roc_op                          = components.load_component_from_url.*|roc_op = components.load_component_from_file('../../components/local/roc/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|kubeflow_deploy_op              = components.load_component_from_url.*|kubeflow_deploy_op = components.load_component_from_file('../../components/kubeflow/deployer/component.yaml')|g" taxi-cab-classification-pipeline.py
   fi
 
   dsl-compile --py taxi-cab-classification-pipeline.py --output taxi-cab-classification-pipeline.zip
@@ -275,8 +285,14 @@ elif [ "$TEST_NAME" == "xgboost" ]; then
     sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_TRANSFORM_IMAGE}|g" xgboost-training-cm.py
     sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-train:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_TRAIN_IMAGE}|g" xgboost-training-cm.py
     sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_PREDICT_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" xgboost-training-cm.py
+
+    # Update the image tag in the yaml.
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" ../../components/local/confusion_matrix/component.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" ../../components/local/roc/component.yaml
+
+    # Update the yaml reference in the sample
+    sed -i -e "s|confusion_matrix_op = components.load_component_from_url.*|confusion_matrix_op = components.load_component_from_file('../../components/local/confusion_matrix/component.yaml')|g" xgboost-training-cm.py
+    sed -i -e "s|roc_op =              components.load_component_from_url.*|roc_op = components.load_component_from_file('../../components/local/roc/component.yaml')|g" xgboost-training-cm.py
   fi
   dsl-compile --py xgboost-training-cm.py --output xgboost-training-cm.zip
 

From 84b071c4e985ff753d0fbad4c8ba6be0cdce38fb Mon Sep 17 00:00:00 2001
From: Ning Gao <ngao@google.com>
Date: Thu, 2 May 2019 12:29:01 -0700
Subject: [PATCH 2/2] replace the image tag in the yaml

---
 test/sample-test/run_test.sh | 58 ++++++++++++++----------------------
 1 file changed, 22 insertions(+), 36 deletions(-)

diff --git a/test/sample-test/run_test.sh b/test/sample-test/run_test.sh
index dc0348bd0f3..6f524c4db56 100755
--- a/test/sample-test/run_test.sh
+++ b/test/sample-test/run_test.sh
@@ -167,30 +167,22 @@ elif [ "$TEST_NAME" == "tfx" ]; then
   # Compile samples
   cd ${BASE_DIR}/samples/tfx
 
+  dsl-compile --py taxi-cab-classification-pipeline.py --output taxi-cab-classification-pipeline.yaml
+
   if [ -n "${DATAFLOW_TFT_IMAGE}" ];then
     # Update the image tag in the yaml.
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFT_IMAGE}|g" ../../components/dataflow/tft/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_PREDICT_IMAGE}|g" ../../components/dataflow/predict/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFDV_IMAGE}|g" ../../components/dataflow/tfdv/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFMA_IMAGE}|g" ../../components/dataflow/tfma/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DNNTRAINER_IMAGE}|g" ../../components/kubeflow/dnntrainer/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DEPLOYER_IMAGE}|g" ../../components/kubeflow/deployer/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" ../../components/local/confusion_matrix/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" ../../components/local/roc/component.yaml
-    # Update the yaml reference in the sample
-    sed -i -e "s|dataflow_tf_data_validation_op  = components.load_component_from_url.*|dataflow_tf_data_validation_op = components.load_component_from_file('../../components/dataflow/tfdv/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|dataflow_tf_transform_op        = components.load_component_from_url.*|dataflow_tf_transform_op = components.load_component_from_file('../../components/dataflow/tft/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|tf_train_op                     = components.load_component_from_url.*|tf_train_op = components.load_component_from_file('../../components/kubeflow/dnntrainer/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|dataflow_tf_model_analyze_op    = components.load_component_from_url.*|dataflow_tf_model_analyze_op = components.load_component_from_file('../../components/dataflow/tfma/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|dataflow_tf_predict_op          = components.load_component_from_url.*|dataflow_tf_predict_op = components.load_component_from_file('../../components/dataflow/predict/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|confusion_matrix_op             = components.load_component_from_url.*|confusion_matrix_op = components.load_component_from_file('../../components/local/confusion_matrix/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|roc_op                          = components.load_component_from_url.*|roc_op = components.load_component_from_file('../../components/local/roc/component.yaml')|g" taxi-cab-classification-pipeline.py
-    sed -i -e "s|kubeflow_deploy_op              = components.load_component_from_url.*|kubeflow_deploy_op = components.load_component_from_file('../../components/kubeflow/deployer/component.yaml')|g" taxi-cab-classification-pipeline.py
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFT_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_PREDICT_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFDV_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:\([a-zA-Z0-9_.-]\)\+|${DATAFLOW_TFMA_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DNNTRAINER_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:\([a-zA-Z0-9_.-]\)\+|${KUBEFLOW_DEPLOYER_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" taxi-cab-classification-pipeline.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" taxi-cab-classification-pipeline.yaml
   fi
 
-  dsl-compile --py taxi-cab-classification-pipeline.py --output taxi-cab-classification-pipeline.zip
   cd "${TEST_DIR}"
-  python3 run_tfx_test.py --input ${BASE_DIR}/samples/tfx/taxi-cab-classification-pipeline.zip --result $SAMPLE_TFX_TEST_RESULT --output $SAMPLE_TFX_TEST_OUTPUT --namespace ${NAMESPACE}
+  python3 run_tfx_test.py --input ${BASE_DIR}/samples/tfx/taxi-cab-classification-pipeline.yaml --result $SAMPLE_TFX_TEST_RESULT --output $SAMPLE_TFX_TEST_OUTPUT --namespace ${NAMESPACE}
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_TFX_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_TFX_TEST_RESULT}
 elif [ "$TEST_NAME" == "sequential" ]; then
@@ -278,26 +270,20 @@ elif [ "$TEST_NAME" == "xgboost" ]; then
   # Compile samples
   cd ${BASE_DIR}/samples/xgboost-spark
 
-  if [ -n "${DATAPROC_CREATE_CLUSTER_IMAGE}" ];then
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_CREATE_CLUSTER_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_DELETE_CLUSTER_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_ANALYZE_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_TRANSFORM_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-train:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_TRAIN_IMAGE}|g" xgboost-training-cm.py
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_PREDICT_IMAGE}|g" xgboost-training-cm.py
-
-    # Update the image tag in the yaml.
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" ../../components/local/confusion_matrix/component.yaml
-    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" ../../components/local/roc/component.yaml
+  dsl-compile --py xgboost-training-cm.py --output xgboost-training-cm.yaml
 
-    # Update the yaml reference in the sample
-    sed -i -e "s|confusion_matrix_op = components.load_component_from_url.*|confusion_matrix_op = components.load_component_from_file('../../components/local/confusion_matrix/component.yaml')|g" xgboost-training-cm.py
-    sed -i -e "s|roc_op =              components.load_component_from_url.*|roc_op = components.load_component_from_file('../../components/local/roc/component.yaml')|g" xgboost-training-cm.py
+  if [ -n "${DATAPROC_CREATE_CLUSTER_IMAGE}" ];then
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_CREATE_CLUSTER_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_DELETE_CLUSTER_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_ANALYZE_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_TRANSFORM_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-train:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_TRAIN_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:\([a-zA-Z0-9_.-]\)\+|${DATAPROC_PREDICT_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:\([a-zA-Z0-9_.-]\)\+|${LOCAL_CONFUSIONMATRIX_IMAGE}|g" xgboost-training-cm.yaml
+    sed -i -e "s|gcr.io/ml-pipeline/ml-pipeline-local-roc:\([a-zA-Z0-9_.-]\)\+|${LOCAL_ROC_IMAGE}|g" xgboost-training-cm.yaml
   fi
-  dsl-compile --py xgboost-training-cm.py --output xgboost-training-cm.zip
-
   cd "${TEST_DIR}"
-  python3 run_xgboost_test.py --input ${BASE_DIR}/samples/xgboost-spark/xgboost-training-cm.zip --result $SAMPLE_XGBOOST_TEST_RESULT --output $SAMPLE_XGBOOST_TEST_OUTPUT --namespace ${NAMESPACE}
+  python3 run_xgboost_test.py --input ${BASE_DIR}/samples/xgboost-spark/xgboost-training-cm.yaml --result $SAMPLE_XGBOOST_TEST_RESULT --output $SAMPLE_XGBOOST_TEST_OUTPUT --namespace ${NAMESPACE}
 
   echo "Copy the test results to GCS ${RESULTS_GCS_DIR}/"
   gsutil cp ${SAMPLE_XGBOOST_TEST_RESULT} ${RESULTS_GCS_DIR}/${SAMPLE_XGBOOST_TEST_RESULT}