From 1f153c489eb0d3149a6c0955bfa970ef5041c2a4 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 11 Apr 2019 13:07:13 -0700 Subject: [PATCH 1/2] Removed the use of undocumented compiler API See https://github.com/tensorflow/tfx/pull/30#issuecomment-482289948 The KFP documentation clearly says that the pipeline function must be decorated with `@pipeline` decorator. It's even mentioned in the `compile` function documentation: https://github.com/kubeflow/pipelines/blob/086d4763d9f163acdeb423bc2bc49ab470442a92/sdk/python/kfp/compiler/compiler.py#L636 Running `compile` without having the function decorated https://github.com/tensorflow/tfx/blob/d9f87b9b03ab8bb98653b8a96d5bf952bc22b2b7/tfx/orchestration/kubeflow/runner.py#L125 will raise error: https://github.com/kubeflow/pipelines/blob/086d4763d9f163acdeb423bc2bc49ab470442a92/sdk/python/kfp/compiler/compiler.py#L578 `'Please use a function with @dsl.pipeline decorator.'`. For some reason this documented behavior was circumvented by a hack that used an undocumented function that should be a part of internal compiler API. This PR fixes this issue. --- tfx/orchestration/kubeflow/runner.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tfx/orchestration/kubeflow/runner.py b/tfx/orchestration/kubeflow/runner.py index bd767e4a4d..1004e18a7c 100644 --- a/tfx/orchestration/kubeflow/runner.py +++ b/tfx/orchestration/kubeflow/runner.py @@ -94,6 +94,10 @@ def run(self, pipeline): pipeline. """ + @dsl.pipeline( + pipeline.pipeline_args['pipeline_name'], + pipeline.pipeline_args.get('description', '') + ) def _construct_pipeline(): """Constructs a Kubeflow pipeline. @@ -116,10 +120,5 @@ def _construct_pipeline(): self._construct_pipeline_graph(pipeline) - pipeline_name = pipeline.pipeline_args['pipeline_name'] - dsl.Pipeline.add_pipeline(pipeline_name, - pipeline.pipeline_args.get('description', ''), - _construct_pipeline) - pipeline_file = os.path.join(self._output_dir, pipeline_name + '.tar.gz') compiler.Compiler().compile(_construct_pipeline, pipeline_file) From 42ab48150f82545d6cd48305ba5c763551ad875f Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 22 Apr 2019 14:26:04 -0700 Subject: [PATCH 2/2] Using explicit parameter names in decorator --- tfx/orchestration/kubeflow/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tfx/orchestration/kubeflow/runner.py b/tfx/orchestration/kubeflow/runner.py index 1004e18a7c..9e0c181e89 100644 --- a/tfx/orchestration/kubeflow/runner.py +++ b/tfx/orchestration/kubeflow/runner.py @@ -95,8 +95,8 @@ def run(self, pipeline): """ @dsl.pipeline( - pipeline.pipeline_args['pipeline_name'], - pipeline.pipeline_args.get('description', '') + name=pipeline.pipeline_args['pipeline_name'], + description=pipeline.pipeline_args.get('description', '') ) def _construct_pipeline(): """Constructs a Kubeflow pipeline.