From 732bb7a0b3e6accce5a62fba933915fa957607d1 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Fri, 21 Jun 2019 10:32:47 -0700 Subject: [PATCH 1/3] Add gcp-connector label and remove pod name --- components/gcp/bigquery/query/component.yaml | 5 +++-- components/gcp/dataflow/launch_python/component.yaml | 5 +++-- components/gcp/dataflow/launch_template/component.yaml | 5 +++-- components/gcp/dataproc/create_cluster/component.yaml | 5 +++-- components/gcp/dataproc/delete_cluster/component.yaml | 7 ++++--- components/gcp/dataproc/submit_hadoop_job/component.yaml | 5 +++-- components/gcp/dataproc/submit_hive_job/component.yaml | 5 +++-- components/gcp/dataproc/submit_pig_job/component.yaml | 5 +++-- components/gcp/dataproc/submit_pyspark_job/component.yaml | 5 +++-- components/gcp/dataproc/submit_spark_job/component.yaml | 5 +++-- components/gcp/dataproc/submit_sparksql_job/component.yaml | 5 +++-- components/gcp/ml_engine/batch_predict/component.yaml | 5 +++-- components/gcp/ml_engine/deploy/component.yaml | 5 +++-- components/gcp/ml_engine/train/component.yaml | 5 +++-- 14 files changed, 43 insertions(+), 29 deletions(-) diff --git a/components/gcp/bigquery/query/component.yaml b/components/gcp/bigquery/query/component.yaml index 816ac748e07..d326ddd1c50 100644 --- a/components/gcp/bigquery/query/component.yaml +++ b/components/gcp/bigquery/query/component.yaml @@ -16,6 +16,9 @@ name: Bigquery - Query description: | A Kubeflow Pipeline component to submit a query to Google Cloud Bigquery service and dump outputs to a Google Cloud Storage blob. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: query description: 'The query used by Bigquery service to fetch the results.' @@ -64,7 +67,5 @@ implementation: --output_gcs_path, {inputValue: output_gcs_path}, --job_config, {inputValue: job_config} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: output_gcs_path: /tmp/kfp/output/bigquery/query-output-path.txt \ No newline at end of file diff --git a/components/gcp/dataflow/launch_python/component.yaml b/components/gcp/dataflow/launch_python/component.yaml index 6f1ab479925..89f9f6a3afc 100644 --- a/components/gcp/dataflow/launch_python/component.yaml +++ b/components/gcp/dataflow/launch_python/component.yaml @@ -15,6 +15,9 @@ name: Launch Python description: | Launch a self-executing beam python file. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: python_file_path description: 'The gcs or local path to the python file to run.' @@ -58,7 +61,5 @@ implementation: --args, {inputValue: args}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataflow/launch_template/component.yaml b/components/gcp/dataflow/launch_template/component.yaml index 601fd3e1462..df4983a2480 100644 --- a/components/gcp/dataflow/launch_template/component.yaml +++ b/components/gcp/dataflow/launch_template/component.yaml @@ -15,6 +15,9 @@ name: Launch Dataflow Template description: | Launchs a dataflow job from template. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: 'Required. The ID of the Cloud Platform project that the job belongs to.' @@ -69,7 +72,5 @@ implementation: --staging_dir, {inputValue: staging_dir}, --wait_interval, {inputValue: wait_interval}, ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/create_cluster/component.yaml b/components/gcp/dataproc/create_cluster/component.yaml index d49f93facb4..234e8c1144b 100644 --- a/components/gcp/dataproc/create_cluster/component.yaml +++ b/components/gcp/dataproc/create_cluster/component.yaml @@ -15,6 +15,9 @@ name: dataproc_create_cluster description: | Creates a DataProc cluster under a project. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -78,7 +81,5 @@ implementation: --cluster, {inputValue: cluster}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: cluster_name: /tmp/kfp/output/dataproc/cluster_name.txt \ No newline at end of file diff --git a/components/gcp/dataproc/delete_cluster/component.yaml b/components/gcp/dataproc/delete_cluster/component.yaml index fcc14149fe7..e6449506412 100644 --- a/components/gcp/dataproc/delete_cluster/component.yaml +++ b/components/gcp/dataproc/delete_cluster/component.yaml @@ -15,6 +15,9 @@ name: dataproc_delete_cluster description: | Deletes a DataProc cluster. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -40,6 +43,4 @@ implementation: --region, {inputValue: region}, --name, {inputValue: name}, --wait_interval, {inputValue: wait_interval} - ] - env: - KFP_POD_NAME: "{{pod.name}}" \ No newline at end of file + ] \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hadoop_job/component.yaml b/components/gcp/dataproc/submit_hadoop_job/component.yaml index e5aada34f93..d7096ea0fb7 100644 --- a/components/gcp/dataproc/submit_hadoop_job/component.yaml +++ b/components/gcp/dataproc/submit_hadoop_job/component.yaml @@ -16,6 +16,9 @@ name: dataproc_submit_hadoop_job description: >- Submits a Cloud Dataproc job for running Apache Hadoop MapReduce jobs on Apache Hadoop YARN. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -88,7 +91,5 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hive_job/component.yaml b/components/gcp/dataproc/submit_hive_job/component.yaml index 09f2712a393..66e4784ad06 100644 --- a/components/gcp/dataproc/submit_hive_job/component.yaml +++ b/components/gcp/dataproc/submit_hive_job/component.yaml @@ -15,6 +15,9 @@ name: dataproc_submit_hive_job description: >- Submits a Cloud Dataproc job for running Apache Hive queries on YARN. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -83,7 +86,5 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pig_job/component.yaml b/components/gcp/dataproc/submit_pig_job/component.yaml index 775a343202b..fe7350d4a63 100644 --- a/components/gcp/dataproc/submit_pig_job/component.yaml +++ b/components/gcp/dataproc/submit_pig_job/component.yaml @@ -15,6 +15,9 @@ name: dataproc_submit_pig_job description: >- Submits a Cloud Dataproc job for running Apache Pig queries on YARN. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -83,7 +86,5 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pyspark_job/component.yaml b/components/gcp/dataproc/submit_pyspark_job/component.yaml index 1c56a974e39..97aa7fe0570 100644 --- a/components/gcp/dataproc/submit_pyspark_job/component.yaml +++ b/components/gcp/dataproc/submit_pyspark_job/component.yaml @@ -15,6 +15,9 @@ name: dataproc_submit_pyspark_job description: >- Submits a Cloud Dataproc job for running Apache PySpark applications on YARN. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -76,7 +79,5 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_spark_job/component.yaml b/components/gcp/dataproc/submit_spark_job/component.yaml index 40238729c10..4dcad090eee 100644 --- a/components/gcp/dataproc/submit_spark_job/component.yaml +++ b/components/gcp/dataproc/submit_spark_job/component.yaml @@ -15,6 +15,9 @@ name: dataproc_submit_spark_job description: >- Submits a Cloud Dataproc job for running Apache Spark applications on YARN. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -84,7 +87,5 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_sparksql_job/component.yaml b/components/gcp/dataproc/submit_sparksql_job/component.yaml index 4da28ffc97f..e01492737c8 100644 --- a/components/gcp/dataproc/submit_sparksql_job/component.yaml +++ b/components/gcp/dataproc/submit_sparksql_job/component.yaml @@ -15,6 +15,9 @@ name: dataproc_submit_sparksql_job description: >- Submits a Cloud Dataproc job for running Apache Spark SQL queries. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: >- @@ -83,7 +86,5 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/ml_engine/batch_predict/component.yaml b/components/gcp/ml_engine/batch_predict/component.yaml index 5c153096ba6..a111bbccb4b 100644 --- a/components/gcp/ml_engine/batch_predict/component.yaml +++ b/components/gcp/ml_engine/batch_predict/component.yaml @@ -15,6 +15,9 @@ name: Batch predict against a model with Cloud ML Engine description: | Creates a MLEngine batch prediction job. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: 'Required. The ID of the parent project of the job.' @@ -78,7 +81,5 @@ implementation: --job_id_prefix, {inputValue: job_id_prefix}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt \ No newline at end of file diff --git a/components/gcp/ml_engine/deploy/component.yaml b/components/gcp/ml_engine/deploy/component.yaml index b65620e8d66..0e7bd6e460f 100644 --- a/components/gcp/ml_engine/deploy/component.yaml +++ b/components/gcp/ml_engine/deploy/component.yaml @@ -16,6 +16,9 @@ name: Deploying a trained model to Cloud Machine Learning Engine description: | A Kubeflow Pipeline component to deploy a trained model from a Cloud Storage path to a Cloud Machine Learning Engine service. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: model_uri description: >- @@ -103,8 +106,6 @@ implementation: --set_default, {inputValue: set_default}, --wait_interval, {inputValue: wait_interval}, ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: model_uri: /tmp/kfp/output/ml_engine/model_uri.txt model_name: /tmp/kfp/output/ml_engine/model_name.txt diff --git a/components/gcp/ml_engine/train/component.yaml b/components/gcp/ml_engine/train/component.yaml index 2df75ab39ca..2cd7b711c5c 100644 --- a/components/gcp/ml_engine/train/component.yaml +++ b/components/gcp/ml_engine/train/component.yaml @@ -16,6 +16,9 @@ name: Submitting a Cloud ML training job as a pipeline step description: | A Kubeflow Pipeline component to submit a Cloud Machine Learning (Cloud ML) Engine training job as a step in a pipeline. +metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector inputs: - name: project_id description: 'Required. The ID of the parent project of the job.' @@ -115,8 +118,6 @@ implementation: --job_id_prefix, {inputValue: job_id_prefix}, --wait_interval, {inputValue: wait_interval} ] - env: - KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt job_dir: /tmp/kfp/output/ml_engine/job_dir.txt From 0a99ea8bcff824df8bef7b80a06d7936e74579d8 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Thu, 11 Jul 2019 17:00:55 -0700 Subject: [PATCH 2/3] Add add-pod-env label and add pod id back --- components/gcp/bigquery/query/component.yaml | 4 +++- components/gcp/dataflow/launch_python/component.yaml | 4 +++- components/gcp/dataflow/launch_template/component.yaml | 4 +++- components/gcp/dataproc/create_cluster/component.yaml | 4 +++- components/gcp/dataproc/delete_cluster/component.yaml | 6 ++++-- components/gcp/dataproc/submit_hadoop_job/component.yaml | 4 +++- components/gcp/dataproc/submit_hive_job/component.yaml | 4 +++- components/gcp/dataproc/submit_pig_job/component.yaml | 4 +++- components/gcp/dataproc/submit_pyspark_job/component.yaml | 4 +++- components/gcp/dataproc/submit_spark_job/component.yaml | 4 +++- components/gcp/dataproc/submit_sparksql_job/component.yaml | 4 +++- components/gcp/ml_engine/batch_predict/component.yaml | 4 +++- components/gcp/ml_engine/deploy/component.yaml | 4 +++- components/gcp/ml_engine/train/component.yaml | 4 +++- 14 files changed, 43 insertions(+), 15 deletions(-) diff --git a/components/gcp/bigquery/query/component.yaml b/components/gcp/bigquery/query/component.yaml index d326ddd1c50..c268fb47c60 100644 --- a/components/gcp/bigquery/query/component.yaml +++ b/components/gcp/bigquery/query/component.yaml @@ -18,7 +18,7 @@ description: | service and dump outputs to a Google Cloud Storage blob. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: query description: 'The query used by Bigquery service to fetch the results.' @@ -67,5 +67,7 @@ implementation: --output_gcs_path, {inputValue: output_gcs_path}, --job_config, {inputValue: job_config} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: output_gcs_path: /tmp/kfp/output/bigquery/query-output-path.txt \ No newline at end of file diff --git a/components/gcp/dataflow/launch_python/component.yaml b/components/gcp/dataflow/launch_python/component.yaml index 89f9f6a3afc..3a1c22157b4 100644 --- a/components/gcp/dataflow/launch_python/component.yaml +++ b/components/gcp/dataflow/launch_python/component.yaml @@ -17,7 +17,7 @@ description: | Launch a self-executing beam python file. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: python_file_path description: 'The gcs or local path to the python file to run.' @@ -61,5 +61,7 @@ implementation: --args, {inputValue: args}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataflow/launch_template/component.yaml b/components/gcp/dataflow/launch_template/component.yaml index df4983a2480..36ff39eeaa3 100644 --- a/components/gcp/dataflow/launch_template/component.yaml +++ b/components/gcp/dataflow/launch_template/component.yaml @@ -17,7 +17,7 @@ description: | Launchs a dataflow job from template. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: 'Required. The ID of the Cloud Platform project that the job belongs to.' @@ -72,5 +72,7 @@ implementation: --staging_dir, {inputValue: staging_dir}, --wait_interval, {inputValue: wait_interval}, ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/create_cluster/component.yaml b/components/gcp/dataproc/create_cluster/component.yaml index 234e8c1144b..06de7b95745 100644 --- a/components/gcp/dataproc/create_cluster/component.yaml +++ b/components/gcp/dataproc/create_cluster/component.yaml @@ -17,7 +17,7 @@ description: | Creates a DataProc cluster under a project. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -81,5 +81,7 @@ implementation: --cluster, {inputValue: cluster}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: cluster_name: /tmp/kfp/output/dataproc/cluster_name.txt \ No newline at end of file diff --git a/components/gcp/dataproc/delete_cluster/component.yaml b/components/gcp/dataproc/delete_cluster/component.yaml index e6449506412..acad75c10d2 100644 --- a/components/gcp/dataproc/delete_cluster/component.yaml +++ b/components/gcp/dataproc/delete_cluster/component.yaml @@ -17,7 +17,7 @@ description: | Deletes a DataProc cluster. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -43,4 +43,6 @@ implementation: --region, {inputValue: region}, --name, {inputValue: name}, --wait_interval, {inputValue: wait_interval} - ] \ No newline at end of file + ] + env: + KFP_POD_NAME: "{{pod.name}}" \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hadoop_job/component.yaml b/components/gcp/dataproc/submit_hadoop_job/component.yaml index d7096ea0fb7..559cb9ba0b8 100644 --- a/components/gcp/dataproc/submit_hadoop_job/component.yaml +++ b/components/gcp/dataproc/submit_hadoop_job/component.yaml @@ -18,7 +18,7 @@ description: >- Apache Hadoop YARN. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -91,5 +91,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hive_job/component.yaml b/components/gcp/dataproc/submit_hive_job/component.yaml index 66e4784ad06..0d6313ff16c 100644 --- a/components/gcp/dataproc/submit_hive_job/component.yaml +++ b/components/gcp/dataproc/submit_hive_job/component.yaml @@ -17,7 +17,7 @@ description: >- Submits a Cloud Dataproc job for running Apache Hive queries on YARN. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -86,5 +86,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pig_job/component.yaml b/components/gcp/dataproc/submit_pig_job/component.yaml index fe7350d4a63..3f8d9b60039 100644 --- a/components/gcp/dataproc/submit_pig_job/component.yaml +++ b/components/gcp/dataproc/submit_pig_job/component.yaml @@ -17,7 +17,7 @@ description: >- Submits a Cloud Dataproc job for running Apache Pig queries on YARN. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -86,5 +86,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pyspark_job/component.yaml b/components/gcp/dataproc/submit_pyspark_job/component.yaml index 97aa7fe0570..1cde278ec3c 100644 --- a/components/gcp/dataproc/submit_pyspark_job/component.yaml +++ b/components/gcp/dataproc/submit_pyspark_job/component.yaml @@ -17,7 +17,7 @@ description: >- Submits a Cloud Dataproc job for running Apache PySpark applications on YARN. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -79,5 +79,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_spark_job/component.yaml b/components/gcp/dataproc/submit_spark_job/component.yaml index 4dcad090eee..123faa22426 100644 --- a/components/gcp/dataproc/submit_spark_job/component.yaml +++ b/components/gcp/dataproc/submit_spark_job/component.yaml @@ -17,7 +17,7 @@ description: >- Submits a Cloud Dataproc job for running Apache Spark applications on YARN. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -87,5 +87,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_sparksql_job/component.yaml b/components/gcp/dataproc/submit_sparksql_job/component.yaml index e01492737c8..746981a4b07 100644 --- a/components/gcp/dataproc/submit_sparksql_job/component.yaml +++ b/components/gcp/dataproc/submit_sparksql_job/component.yaml @@ -17,7 +17,7 @@ description: >- Submits a Cloud Dataproc job for running Apache Spark SQL queries. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: >- @@ -86,5 +86,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/ml_engine/batch_predict/component.yaml b/components/gcp/ml_engine/batch_predict/component.yaml index a111bbccb4b..375b027c614 100644 --- a/components/gcp/ml_engine/batch_predict/component.yaml +++ b/components/gcp/ml_engine/batch_predict/component.yaml @@ -17,7 +17,7 @@ description: | Creates a MLEngine batch prediction job. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: 'Required. The ID of the parent project of the job.' @@ -81,5 +81,7 @@ implementation: --job_id_prefix, {inputValue: job_id_prefix}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt \ No newline at end of file diff --git a/components/gcp/ml_engine/deploy/component.yaml b/components/gcp/ml_engine/deploy/component.yaml index 0e7bd6e460f..10899a93f3a 100644 --- a/components/gcp/ml_engine/deploy/component.yaml +++ b/components/gcp/ml_engine/deploy/component.yaml @@ -18,7 +18,7 @@ description: | path to a Cloud Machine Learning Engine service. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: model_uri description: >- @@ -106,6 +106,8 @@ implementation: --set_default, {inputValue: set_default}, --wait_interval, {inputValue: wait_interval}, ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: model_uri: /tmp/kfp/output/ml_engine/model_uri.txt model_name: /tmp/kfp/output/ml_engine/model_name.txt diff --git a/components/gcp/ml_engine/train/component.yaml b/components/gcp/ml_engine/train/component.yaml index 2cd7b711c5c..e7bf919ea94 100644 --- a/components/gcp/ml_engine/train/component.yaml +++ b/components/gcp/ml_engine/train/component.yaml @@ -18,7 +18,7 @@ description: | Engine training job as a step in a pipeline. metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' inputs: - name: project_id description: 'Required. The ID of the parent project of the job.' @@ -118,6 +118,8 @@ implementation: --job_id_prefix, {inputValue: job_id_prefix}, --wait_interval, {inputValue: wait_interval} ] + env: + KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt job_dir: /tmp/kfp/output/ml_engine/job_dir.txt From 42767e1d7a092d3ff57dbf7f90a2f9ac1ffb561f Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Fri, 12 Jul 2019 09:21:11 -0700 Subject: [PATCH 3/3] Remove space --- components/gcp/bigquery/query/component.yaml | 2 +- components/gcp/dataflow/launch_python/component.yaml | 2 +- components/gcp/dataflow/launch_template/component.yaml | 2 +- components/gcp/dataproc/create_cluster/component.yaml | 2 +- components/gcp/dataproc/delete_cluster/component.yaml | 2 +- components/gcp/dataproc/submit_hadoop_job/component.yaml | 2 +- components/gcp/dataproc/submit_hive_job/component.yaml | 2 +- components/gcp/dataproc/submit_pig_job/component.yaml | 2 +- components/gcp/dataproc/submit_pyspark_job/component.yaml | 2 +- components/gcp/dataproc/submit_spark_job/component.yaml | 2 +- components/gcp/dataproc/submit_sparksql_job/component.yaml | 2 +- components/gcp/ml_engine/batch_predict/component.yaml | 2 +- components/gcp/ml_engine/deploy/component.yaml | 2 +- components/gcp/ml_engine/train/component.yaml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/components/gcp/bigquery/query/component.yaml b/components/gcp/bigquery/query/component.yaml index 19648f4b064..fed32aef68d 100644 --- a/components/gcp/bigquery/query/component.yaml +++ b/components/gcp/bigquery/query/component.yaml @@ -68,7 +68,7 @@ implementation: --output_gcs_path, {inputValue: output_gcs_path}, --job_config, {inputValue: job_config} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: output_gcs_path: /tmp/kfp/output/bigquery/query-output-path.txt diff --git a/components/gcp/dataflow/launch_python/component.yaml b/components/gcp/dataflow/launch_python/component.yaml index d97f5c62192..3d14e24d7d1 100644 --- a/components/gcp/dataflow/launch_python/component.yaml +++ b/components/gcp/dataflow/launch_python/component.yaml @@ -61,7 +61,7 @@ implementation: --args, {inputValue: args}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataflow/launch_template/component.yaml b/components/gcp/dataflow/launch_template/component.yaml index 4ba26e3c0e4..3e7efae7a66 100644 --- a/components/gcp/dataflow/launch_template/component.yaml +++ b/components/gcp/dataflow/launch_template/component.yaml @@ -72,7 +72,7 @@ implementation: --staging_dir, {inputValue: staging_dir}, --wait_interval, {inputValue: wait_interval}, ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/create_cluster/component.yaml b/components/gcp/dataproc/create_cluster/component.yaml index 494658573b7..b009ab3d541 100644 --- a/components/gcp/dataproc/create_cluster/component.yaml +++ b/components/gcp/dataproc/create_cluster/component.yaml @@ -81,7 +81,7 @@ implementation: --cluster, {inputValue: cluster}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: cluster_name: /tmp/kfp/output/dataproc/cluster_name.txt \ No newline at end of file diff --git a/components/gcp/dataproc/delete_cluster/component.yaml b/components/gcp/dataproc/delete_cluster/component.yaml index 44a48f0595e..452d89ad73b 100644 --- a/components/gcp/dataproc/delete_cluster/component.yaml +++ b/components/gcp/dataproc/delete_cluster/component.yaml @@ -44,5 +44,5 @@ implementation: --name, {inputValue: name}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hadoop_job/component.yaml b/components/gcp/dataproc/submit_hadoop_job/component.yaml index 68b8c417e15..281d65d8cbe 100644 --- a/components/gcp/dataproc/submit_hadoop_job/component.yaml +++ b/components/gcp/dataproc/submit_hadoop_job/component.yaml @@ -91,7 +91,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_hive_job/component.yaml b/components/gcp/dataproc/submit_hive_job/component.yaml index 3813251fad7..558179dedfe 100644 --- a/components/gcp/dataproc/submit_hive_job/component.yaml +++ b/components/gcp/dataproc/submit_hive_job/component.yaml @@ -86,7 +86,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pig_job/component.yaml b/components/gcp/dataproc/submit_pig_job/component.yaml index a387a739e86..61fcc879627 100644 --- a/components/gcp/dataproc/submit_pig_job/component.yaml +++ b/components/gcp/dataproc/submit_pig_job/component.yaml @@ -86,7 +86,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_pyspark_job/component.yaml b/components/gcp/dataproc/submit_pyspark_job/component.yaml index e34d3decbe1..e67f28b46c2 100644 --- a/components/gcp/dataproc/submit_pyspark_job/component.yaml +++ b/components/gcp/dataproc/submit_pyspark_job/component.yaml @@ -79,7 +79,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_spark_job/component.yaml b/components/gcp/dataproc/submit_spark_job/component.yaml index 28515a536a9..e73058f7585 100644 --- a/components/gcp/dataproc/submit_spark_job/component.yaml +++ b/components/gcp/dataproc/submit_spark_job/component.yaml @@ -87,7 +87,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/dataproc/submit_sparksql_job/component.yaml b/components/gcp/dataproc/submit_sparksql_job/component.yaml index 7806d8146b3..405e5405ca1 100644 --- a/components/gcp/dataproc/submit_sparksql_job/component.yaml +++ b/components/gcp/dataproc/submit_sparksql_job/component.yaml @@ -86,7 +86,7 @@ implementation: --job, {inputValue: job}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file diff --git a/components/gcp/ml_engine/batch_predict/component.yaml b/components/gcp/ml_engine/batch_predict/component.yaml index f14fa9eccc5..f3f263a3e3d 100644 --- a/components/gcp/ml_engine/batch_predict/component.yaml +++ b/components/gcp/ml_engine/batch_predict/component.yaml @@ -81,7 +81,7 @@ implementation: --job_id_prefix, {inputValue: job_id_prefix}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt \ No newline at end of file diff --git a/components/gcp/ml_engine/deploy/component.yaml b/components/gcp/ml_engine/deploy/component.yaml index 6636836a4f6..237cd792d26 100644 --- a/components/gcp/ml_engine/deploy/component.yaml +++ b/components/gcp/ml_engine/deploy/component.yaml @@ -106,7 +106,7 @@ implementation: --set_default, {inputValue: set_default}, --wait_interval, {inputValue: wait_interval}, ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: model_uri: /tmp/kfp/output/ml_engine/model_uri.txt diff --git a/components/gcp/ml_engine/train/component.yaml b/components/gcp/ml_engine/train/component.yaml index ce026094815..d85149599ed 100644 --- a/components/gcp/ml_engine/train/component.yaml +++ b/components/gcp/ml_engine/train/component.yaml @@ -118,7 +118,7 @@ implementation: --job_id_prefix, {inputValue: job_id_prefix}, --wait_interval, {inputValue: wait_interval} ] - env: + env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt