From df64ab2a1b8ac968d19218a9552cded7f3562b23 Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 13 Sep 2019 13:15:35 -0700 Subject: [PATCH 1/9] add a new convention --- samples/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/samples/README.md b/samples/README.md index 9a743be29ec..0978a1937aa 100644 --- a/samples/README.md +++ b/samples/README.md @@ -75,8 +75,9 @@ For better readability and integrations with the sample test infrastructure, sam * The sample file should be either `*.py` or `*.ipynb`, and its file name is consistent with its directory name. * For `*.py` sample, it's recommended to have a main invoking `kfp.compiler.Compiler().compile()` to compile the pipeline function into pipeline yaml spec. -* For `*.ipynb` sample, parameters (e.g., experiment name and project name) should be defined in a dedicated cell and -tagged as parameter. Detailed guideline is [here](https://github.com/nteract/papermill). Also, all the environment setup and +* For `*.ipynb` sample, parameters, including but not limited to `experiment_name` and `project_name`, +should be defined in a dedicated cell and tagged as parameter. Detailed guideline is +[here](https://github.com/nteract/papermill). Also, all the environment setup and preparation should be within the notebook, such as by `!pip install packages` From 6c8daceb2f19d45964b530ab5402b55157a27841 Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 13 Sep 2019 13:25:18 -0700 Subject: [PATCH 2/9] Fix config experiment_name --- test/sample-test/sample_test_launcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sample-test/sample_test_launcher.py b/test/sample-test/sample_test_launcher.py index 6e4b0d952c5..f9b02c51d77 100644 --- a/test/sample-test/sample_test_launcher.py +++ b/test/sample-test/sample_test_launcher.py @@ -101,7 +101,6 @@ def _compile(self): if self._is_notebook: # Parse necessary params from config.yaml nb_params = {} - try: config_file = os.path.join(CONFIG_DIR, '%s.config.yaml' % self._test_name) with open(config_file, 'r') as f: @@ -119,6 +118,8 @@ def _compile(self): nb_params['output'] = self._sample_test_output if 'run_pipeline' in raw_args.keys(): self._run_pipeline = raw_args['run_pipeline'] + if self._run_pipeline: + nb_params['experiment_name'] = self._test_name + '-test' pm.execute_notebook( input_path='%s.ipynb' % self._test_name, From c24a449ef0665a58a8c35d168130aace235b403e Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 13 Sep 2019 13:30:09 -0700 Subject: [PATCH 3/9] fix light weight example --- .../core/lightweight_component/lightweight_component.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/core/lightweight_component/lightweight_component.ipynb b/samples/core/lightweight_component/lightweight_component.ipynb index a932d19aeab..79db4e6c165 100644 --- a/samples/core/lightweight_component/lightweight_component.ipynb +++ b/samples/core/lightweight_component/lightweight_component.ipynb @@ -29,7 +29,7 @@ }, "outputs": [], "source": [ - "EXPERIMENT_NAME = 'lightweight python components'" + "experiment_name = 'lightweight python components'" ] }, { @@ -263,7 +263,7 @@ "#Get or create an experiment and submit a pipeline run\n", "import kfp\n", "client = kfp.Client()\n", - "experiment = client.create_experiment(EXPERIMENT_NAME)\n", + "experiment = client.create_experiment(experiment_name)\n", "\n", "#Submit a pipeline run\n", "run_name = pipeline_func.__name__ + ' run'\n", From 233a1ed5cb8e08645a10e2fafa6c5c43782b7b68 Mon Sep 17 00:00:00 2001 From: numerology Date: Fri, 13 Sep 2019 14:22:56 -0700 Subject: [PATCH 4/9] Fix experiment name --- test/sample-test/sample_test_launcher.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/sample-test/sample_test_launcher.py b/test/sample-test/sample_test_launcher.py index f9b02c51d77..3215466436d 100644 --- a/test/sample-test/sample_test_launcher.py +++ b/test/sample-test/sample_test_launcher.py @@ -118,8 +118,9 @@ def _compile(self): nb_params['output'] = self._sample_test_output if 'run_pipeline' in raw_args.keys(): self._run_pipeline = raw_args['run_pipeline'] - if self._run_pipeline: - nb_params['experiment_name'] = self._test_name + '-test' + + if self._run_pipeline: + nb_params['experiment_name'] = self._test_name + '-test' pm.execute_notebook( input_path='%s.ipynb' % self._test_name, From 80b577995447155041b5f7a0d78231566e310c01 Mon Sep 17 00:00:00 2001 From: numerology Date: Mon, 16 Sep 2019 13:31:28 -0700 Subject: [PATCH 5/9] Update guidelin. --- samples/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/samples/README.md b/samples/README.md index 0978a1937aa..0a0c39afa06 100644 --- a/samples/README.md +++ b/samples/README.md @@ -75,8 +75,11 @@ For better readability and integrations with the sample test infrastructure, sam * The sample file should be either `*.py` or `*.ipynb`, and its file name is consistent with its directory name. * For `*.py` sample, it's recommended to have a main invoking `kfp.compiler.Compiler().compile()` to compile the pipeline function into pipeline yaml spec. -* For `*.ipynb` sample, parameters, including but not limited to `experiment_name` and `project_name`, -should be defined in a dedicated cell and tagged as parameter. Detailed guideline is +* For `*.ipynb` sample, parameters (e.g., experiment name and project name) +should be defined in a dedicated cell and tagged as parameter. +(If the author would like to let the sample test infra run it, it's necessary to parameterize +experiment name and project name used.) +Detailed guideline is [here](https://github.com/nteract/papermill). Also, all the environment setup and preparation should be within the notebook, such as by `!pip install packages` From 8343a1cbb7dd0e335c9127d87cf4c63991260f1a Mon Sep 17 00:00:00 2001 From: numerology Date: Mon, 16 Sep 2019 13:32:32 -0700 Subject: [PATCH 6/9] fix lightweight_component.config.yaml --- test/sample-test/configs/lightweight_component.config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sample-test/configs/lightweight_component.config.yaml b/test/sample-test/configs/lightweight_component.config.yaml index 23911575245..93578445ad3 100644 --- a/test/sample-test/configs/lightweight_component.config.yaml +++ b/test/sample-test/configs/lightweight_component.config.yaml @@ -14,4 +14,4 @@ test_name: lightweight_component notebook_params: - EXPERIMENT_NAME: lightweight_component-test + experiment_name: lightweight_component-test From e4cd668dc7ccb6ca7201960348c80698688c644e Mon Sep 17 00:00:00 2001 From: numerology Date: Mon, 16 Sep 2019 13:55:24 -0700 Subject: [PATCH 7/9] Remove config yaml --- .../configs/lightweight_component.config.yaml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 test/sample-test/configs/lightweight_component.config.yaml diff --git a/test/sample-test/configs/lightweight_component.config.yaml b/test/sample-test/configs/lightweight_component.config.yaml deleted file mode 100644 index 93578445ad3..00000000000 --- a/test/sample-test/configs/lightweight_component.config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -test_name: lightweight_component -notebook_params: - experiment_name: lightweight_component-test From a1debcdfd0e60eb46368a0177a50f6efe520e8c5 Mon Sep 17 00:00:00 2001 From: numerology Date: Mon, 16 Sep 2019 15:41:37 -0700 Subject: [PATCH 8/9] Fix guideline.md --- samples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/README.md b/samples/README.md index 0a0c39afa06..0360334b1d1 100644 --- a/samples/README.md +++ b/samples/README.md @@ -78,7 +78,7 @@ pipeline function into pipeline yaml spec. * For `*.ipynb` sample, parameters (e.g., experiment name and project name) should be defined in a dedicated cell and tagged as parameter. (If the author would like to let the sample test infra run it, it's necessary to parameterize -experiment name and project name used.) +experiment name used.) Detailed guideline is [here](https://github.com/nteract/papermill). Also, all the environment setup and preparation should be within the notebook, such as by `!pip install packages` From 0cf6fa738bdf92ccf32c97f2c0c16e2bcdb43308 Mon Sep 17 00:00:00 2001 From: numerology Date: Tue, 17 Sep 2019 10:41:43 -0700 Subject: [PATCH 9/9] Reword. --- samples/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/README.md b/samples/README.md index 0360334b1d1..687c1a757db 100644 --- a/samples/README.md +++ b/samples/README.md @@ -75,10 +75,11 @@ For better readability and integrations with the sample test infrastructure, sam * The sample file should be either `*.py` or `*.ipynb`, and its file name is consistent with its directory name. * For `*.py` sample, it's recommended to have a main invoking `kfp.compiler.Compiler().compile()` to compile the pipeline function into pipeline yaml spec. -* For `*.ipynb` sample, parameters (e.g., experiment name and project name) +* For `*.ipynb` sample, parameters (e.g., `experiment_name` and `project_name`) should be defined in a dedicated cell and tagged as parameter. -(If the author would like to let the sample test infra run it, it's necessary to parameterize -experiment name used.) +(If the author would like the sample test infra to run it by setting the `run_pipeline` flag to True in +the associated `config.yaml` file, the sample test infra will expect a parameter `experiment_name` +to inject so that it can run in the sample test experiment.) Detailed guideline is [here](https://github.com/nteract/papermill). Also, all the environment setup and preparation should be within the notebook, such as by `!pip install packages`