From c23c0ddb6efeec1524f3d32b2d92c6b3e875b690 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Thu, 20 Jun 2019 22:46:12 -0700 Subject: [PATCH 1/5] Configure gcp connectors in dsl --- sdk/python/kfp/gcp.py | 28 ++++++++++- sdk/python/tests/compiler/compiler_tests.py | 3 ++ sdk/python/tests/compiler/testdata/gcp.py | 34 +++++++++++++ sdk/python/tests/compiler/testdata/gcp.yaml | 54 +++++++++++++++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 sdk/python/tests/compiler/testdata/gcp.py create mode 100644 sdk/python/tests/compiler/testdata/gcp.yaml diff --git a/sdk/python/kfp/gcp.py b/sdk/python/kfp/gcp.py index 6e1cf30273d..aafb183f32b 100644 --- a/sdk/python/kfp/gcp.py +++ b/sdk/python/kfp/gcp.py @@ -107,4 +107,30 @@ def _set_preemptible(task): task.add_node_selector_constraint("cloud.google.com/gke-preemptible", "true") return task - return _set_preemptible \ No newline at end of file + return _set_preemptible + +def set_gcp_settings(secret_name='user-gcp-sa'): + def _set_gcp_settings(op): + if op.pod_labels and op.pod_labels['pipelines.kubeflow.org/component-type'] == 'gcp-connector': + from kubernetes import client as k8s_client + op.container.add_env_variable( + k8s_client.V1EnvVar( + name='KFP_POD_NAME', + value_from=k8s_client.V1EnvVarSource( + field_ref=k8s_client.V1ObjectFieldSelector( + field_path='metadata.name' + ) + ) + ) + ).add_env_variable( + k8s_client.V1EnvVar( + name='KFP_NAMESPACE', + value_from=k8s_client.V1EnvVarSource( + field_ref=k8s_client.V1ObjectFieldSelector( + field_path='metadata.namespace' + ) + ) + ) + ) + op.apply(use_gcp_secret(secret_name=secret_name)) + return _set_gcp_settings \ No newline at end of file diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index dd16ca15025..ba0c11411aa 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -548,3 +548,6 @@ def some_pipeline(): container = template.get('container', None) if container: self.assertEqual(template['retryStrategy']['limit'], 5) + + def test_gcp_settings(self): + self._test_py_compile_yaml('gcp') diff --git a/sdk/python/tests/compiler/testdata/gcp.py b/sdk/python/tests/compiler/testdata/gcp.py new file mode 100644 index 00000000000..e0d2a65170e --- /dev/null +++ b/sdk/python/tests/compiler/testdata/gcp.py @@ -0,0 +1,34 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kfp +import kfp.dsl as dsl +import kfp.gcp as gcp + +@dsl.pipeline( + name='Test gcp settings', + description='Test gcp settings' +) +def test_gcp_settings(): + op = dsl.ContainerOp( + name='echo', + image='library/bash', + command=['sh', '-c'], + arguments=['echo $KFP_POD_NAME']).add_pod_label( + 'pipelines.kubeflow.org/component-type', 'gcp-connector' + ) + dsl.get_pipeline_conf().add_op_transformer(gcp.set_gcp_settings()) + +if __name__ == '__main__': + kfp.compiler.Compiler().compile(test_gcp_settings, __file__ + '.yaml') \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/gcp.yaml b/sdk/python/tests/compiler/testdata/gcp.yaml new file mode 100644 index 00000000000..759e6f6d1f2 --- /dev/null +++ b/sdk/python/tests/compiler/testdata/gcp.yaml @@ -0,0 +1,54 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: test-gcp-settings- +spec: + arguments: + parameters: [] + entrypoint: test-gcp-settings + serviceAccountName: pipeline-runner + templates: + - container: + args: + - echo $KFP_POD_NAME + command: + - sh + - -c + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/gcp-credentials/user-gcp-sa.json + - name: CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE + value: /secret/gcp-credentials/user-gcp-sa.json + image: library/bash + volumeMounts: + - mountPath: /secret/gcp-credentials + name: gcp-credentials-user-gcp-sa + metadata: + labels: + pipelines.kubeflow.org/component-type: gcp-connector + name: echo + outputs: + artifacts: + - name: mlpipeline-ui-metadata + optional: true + path: /mlpipeline-ui-metadata.json + - name: mlpipeline-metrics + optional: true + path: /mlpipeline-metrics.json + - dag: + tasks: + - name: echo + template: echo + name: test-gcp-settings + volumes: + - name: gcp-credentials-user-gcp-sa + secret: + secretName: user-gcp-sa From 7ddc15d3f311cd3689e3c63bbd01f5ef35478cbf Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Fri, 21 Jun 2019 10:24:54 -0700 Subject: [PATCH 2/5] Make configure_gcp_connector more extensible --- sdk/python/kfp/gcp.py | 14 +++++++++++--- sdk/python/tests/compiler/testdata/gcp.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sdk/python/kfp/gcp.py b/sdk/python/kfp/gcp.py index aafb183f32b..eadfa20af8f 100644 --- a/sdk/python/kfp/gcp.py +++ b/sdk/python/kfp/gcp.py @@ -13,6 +13,8 @@ # limitations under the License. from kubernetes.client import V1Toleration +from typing import Callable, List +from .dsl._container_op import BaseOp def use_gcp_secret(secret_name='user-gcp-sa', secret_file_path_in_volume='/user-gcp-sa.json', volume_name=None, secret_volume_mount_path='/secret/gcp-credentials'): @@ -109,8 +111,11 @@ def _set_preemptible(task): return _set_preemptible -def set_gcp_settings(secret_name='user-gcp-sa'): - def _set_gcp_settings(op): +def configure_gcp_connector( + op_transformers: List[Callable[[BaseOp], BaseOp]] = [use_gcp_secret()]) -> Callable[[BaseOp], BaseOp]: + """Configure GCP settings for GCP connector components. + """ + def _set_gcp_settings(op: BaseOp) -> BaseOp: if op.pod_labels and op.pod_labels['pipelines.kubeflow.org/component-type'] == 'gcp-connector': from kubernetes import client as k8s_client op.container.add_env_variable( @@ -132,5 +137,8 @@ def _set_gcp_settings(op): ) ) ) - op.apply(use_gcp_secret(secret_name=secret_name)) + if op_transformers: + for op_transformer in op_transformers: + op.apply(op_transformer) + return op return _set_gcp_settings \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/gcp.py b/sdk/python/tests/compiler/testdata/gcp.py index e0d2a65170e..1e0750bc0f0 100644 --- a/sdk/python/tests/compiler/testdata/gcp.py +++ b/sdk/python/tests/compiler/testdata/gcp.py @@ -28,7 +28,7 @@ def test_gcp_settings(): arguments=['echo $KFP_POD_NAME']).add_pod_label( 'pipelines.kubeflow.org/component-type', 'gcp-connector' ) - dsl.get_pipeline_conf().add_op_transformer(gcp.set_gcp_settings()) + dsl.get_pipeline_conf().add_op_transformer(gcp.configure_gcp_connector()) if __name__ == '__main__': kfp.compiler.Compiler().compile(test_gcp_settings, __file__ + '.yaml') \ No newline at end of file From cb9b35515b0633427e9346078ec3369b151fb45e Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Tue, 2 Jul 2019 01:19:36 -0700 Subject: [PATCH 3/5] Add add_pod_env op handler. --- .../kfp/compiler/_default_transformers.py | 41 +++++++++++++++++++ sdk/python/kfp/compiler/compiler.py | 5 ++- sdk/python/kfp/gcp.py | 36 +--------------- sdk/python/tests/compiler/compiler_tests.py | 4 +- .../testdata/{gcp.py => add_pod_env.py} | 14 +++---- .../testdata/{gcp.yaml => add_pod_env.yaml} | 19 ++------- 6 files changed, 58 insertions(+), 61 deletions(-) create mode 100644 sdk/python/kfp/compiler/_default_transformers.py rename sdk/python/tests/compiler/testdata/{gcp.py => add_pod_env.py} (70%) rename sdk/python/tests/compiler/testdata/{gcp.yaml => add_pod_env.yaml} (59%) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py new file mode 100644 index 00000000000..8bb336b98e3 --- /dev/null +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -0,0 +1,41 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..dsl._container_op import BaseOp + +def add_pod_env(op: BaseOp) -> BaseOp: + """Adds pod environment info to the operator. + """ + if op.pod_labels and op.pod_labels['add-pod-env'] == 'true': + from kubernetes import client as k8s_client + op.container.add_env_variable( + k8s_client.V1EnvVar( + name='KFP_POD_NAME', + value_from=k8s_client.V1EnvVarSource( + field_ref=k8s_client.V1ObjectFieldSelector( + field_path='metadata.name' + ) + ) + ) + ).add_env_variable( + k8s_client.V1EnvVar( + name='KFP_NAMESPACE', + value_from=k8s_client.V1EnvVarSource( + field_ref=k8s_client.V1ObjectFieldSelector( + field_path='metadata.namespace' + ) + ) + ) + ) + return op \ No newline at end of file diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index 0a8b2df1509..67855bffb96 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -23,6 +23,7 @@ from .. import dsl from ._k8s_helper import K8sHelper from ._op_to_template import _op_to_template +from ._default_transformers import add_pod_env from ..dsl._metadata import TypeMeta, _extract_pipeline_metadata from ..dsl._ops_group import OpsGroup @@ -652,7 +653,9 @@ def _compile(self, pipeline_func): sanitized_ops[sanitized_name] = op p.ops = sanitized_ops - workflow = self._create_pipeline_workflow(args_list_with_defaults, p, p.conf.op_transformers) + op_transformers = [add_pod_env] + op_transformers.extend(p.conf.op_transformers) + workflow = self._create_pipeline_workflow(args_list_with_defaults, p, op_transformers) return workflow def compile(self, pipeline_func, package_path, type_check=True): diff --git a/sdk/python/kfp/gcp.py b/sdk/python/kfp/gcp.py index eadfa20af8f..6e1cf30273d 100644 --- a/sdk/python/kfp/gcp.py +++ b/sdk/python/kfp/gcp.py @@ -13,8 +13,6 @@ # limitations under the License. from kubernetes.client import V1Toleration -from typing import Callable, List -from .dsl._container_op import BaseOp def use_gcp_secret(secret_name='user-gcp-sa', secret_file_path_in_volume='/user-gcp-sa.json', volume_name=None, secret_volume_mount_path='/secret/gcp-credentials'): @@ -109,36 +107,4 @@ def _set_preemptible(task): task.add_node_selector_constraint("cloud.google.com/gke-preemptible", "true") return task - return _set_preemptible - -def configure_gcp_connector( - op_transformers: List[Callable[[BaseOp], BaseOp]] = [use_gcp_secret()]) -> Callable[[BaseOp], BaseOp]: - """Configure GCP settings for GCP connector components. - """ - def _set_gcp_settings(op: BaseOp) -> BaseOp: - if op.pod_labels and op.pod_labels['pipelines.kubeflow.org/component-type'] == 'gcp-connector': - from kubernetes import client as k8s_client - op.container.add_env_variable( - k8s_client.V1EnvVar( - name='KFP_POD_NAME', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path='metadata.name' - ) - ) - ) - ).add_env_variable( - k8s_client.V1EnvVar( - name='KFP_NAMESPACE', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path='metadata.namespace' - ) - ) - ) - ) - if op_transformers: - for op_transformer in op_transformers: - op.apply(op_transformer) - return op - return _set_gcp_settings \ No newline at end of file + return _set_preemptible \ No newline at end of file diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index ba0c11411aa..b8c9da359cc 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -549,5 +549,5 @@ def some_pipeline(): if container: self.assertEqual(template['retryStrategy']['limit'], 5) - def test_gcp_settings(self): - self._test_py_compile_yaml('gcp') + def test_add_pod_env(self): + self._test_py_compile_yaml('add_pod_env') diff --git a/sdk/python/tests/compiler/testdata/gcp.py b/sdk/python/tests/compiler/testdata/add_pod_env.py similarity index 70% rename from sdk/python/tests/compiler/testdata/gcp.py rename to sdk/python/tests/compiler/testdata/add_pod_env.py index 1e0750bc0f0..eb0d65826b3 100644 --- a/sdk/python/tests/compiler/testdata/gcp.py +++ b/sdk/python/tests/compiler/testdata/add_pod_env.py @@ -14,21 +14,19 @@ import kfp import kfp.dsl as dsl -import kfp.gcp as gcp @dsl.pipeline( - name='Test gcp settings', - description='Test gcp settings' + name='Test adding pod env', + description='Test adding pod env' ) -def test_gcp_settings(): +def test_add_pod_env(): op = dsl.ContainerOp( name='echo', image='library/bash', command=['sh', '-c'], arguments=['echo $KFP_POD_NAME']).add_pod_label( - 'pipelines.kubeflow.org/component-type', 'gcp-connector' - ) - dsl.get_pipeline_conf().add_op_transformer(gcp.configure_gcp_connector()) + 'add-pod-env', 'true' + ) if __name__ == '__main__': - kfp.compiler.Compiler().compile(test_gcp_settings, __file__ + '.yaml') \ No newline at end of file + kfp.compiler.Compiler().compile(test_add_pod_env, __file__ + '.yaml') \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/gcp.yaml b/sdk/python/tests/compiler/testdata/add_pod_env.yaml similarity index 59% rename from sdk/python/tests/compiler/testdata/gcp.yaml rename to sdk/python/tests/compiler/testdata/add_pod_env.yaml index 759e6f6d1f2..fb4b114587e 100644 --- a/sdk/python/tests/compiler/testdata/gcp.yaml +++ b/sdk/python/tests/compiler/testdata/add_pod_env.yaml @@ -1,11 +1,11 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: test-gcp-settings- + generateName: test-adding-pod-env- spec: arguments: parameters: [] - entrypoint: test-gcp-settings + entrypoint: test-adding-pod-env serviceAccountName: pipeline-runner templates: - container: @@ -23,17 +23,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - - name: CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE - value: /secret/gcp-credentials/user-gcp-sa.json image: library/bash - volumeMounts: - - mountPath: /secret/gcp-credentials - name: gcp-credentials-user-gcp-sa metadata: labels: - pipelines.kubeflow.org/component-type: gcp-connector + add-pod-env: 'true' name: echo outputs: artifacts: @@ -47,8 +40,4 @@ spec: tasks: - name: echo template: echo - name: test-gcp-settings - volumes: - - name: gcp-credentials-user-gcp-sa - secret: - secretName: user-gcp-sa + name: test-adding-pod-env From fe6cd0ed70ac7e88e8c4bcc1d96b31c8c50ead12 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Tue, 2 Jul 2019 23:40:26 -0700 Subject: [PATCH 4/5] Only apply add_pod_env on ContainerOp --- sdk/python/kfp/compiler/_default_transformers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py index 8bb336b98e3..4e9a347df31 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..dsl._container_op import BaseOp +from ..dsl._container_op import BaseOp, ContainerOp def add_pod_env(op: BaseOp) -> BaseOp: - """Adds pod environment info to the operator. + """Adds pod environment info to ContainerOp. """ - if op.pod_labels and op.pod_labels['add-pod-env'] == 'true': + if isinstance(op, ContainerOp) and op.pod_labels and op.pod_labels['add-pod-env'] == 'true': from kubernetes import client as k8s_client op.container.add_env_variable( k8s_client.V1EnvVar( From 8e5689b916e7426d43c32dfde31ffc3b246e1533 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Wed, 3 Jul 2019 00:15:21 -0700 Subject: [PATCH 5/5] Update license header --- sdk/python/kfp/compiler/_default_transformers.py | 2 +- sdk/python/tests/compiler/testdata/add_pod_env.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/compiler/_default_transformers.py index 4e9a347df31..34711be6b58 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/compiler/_default_transformers.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.py b/sdk/python/tests/compiler/testdata/add_pod_env.py index eb0d65826b3..584498a6a7e 100644 --- a/sdk/python/tests/compiler/testdata/add_pod_env.py +++ b/sdk/python/tests/compiler/testdata/add_pod_env.py @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.