forked from opendatahub-io/data-science-pipelines
-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix special compilation cases for kfp samples (#91)
* fix test cases * add eof newline * fix file restructure * Generalize for special cases Generalize for any nested pipeline or pipeline without a decorator as long as there is a config file with the necessary information to compile the pipelines * Add more clear error throwing also fix test_kfp_samples_report.txt * Check for special pipelines from config file * Add comment
- Loading branch information
1 parent
ebe05e3
commit 6aa7ed3
Showing
5 changed files
with
185 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| pipeline: compose.py | ||
| type: nested | ||
| components: | ||
| - name: save_most_frequent_word | ||
| - name: download_save_most_frequent_word | ||
| --- | ||
| pipeline: basic_no_decorator.py | ||
| type: no_decorator | ||
| components: | ||
| function: save_most_frequent_word | ||
| name: 'Save Most Frequent' | ||
| description: 'Get Most Frequent Word and Save to GCS' | ||
| paramsList: ["message_param", "output_path_param"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Copyright 2020 kubeflow.org | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import os | ||
| import sys | ||
| import shutil | ||
| import zipfile | ||
| import yaml | ||
| import tempfile | ||
| import importlib | ||
| import kfp_tekton.compiler as compiler | ||
| import filecmp | ||
|
|
||
| def _get_yaml_from_zip(zip_file): | ||
| with zipfile.ZipFile(zip_file, 'r') as zip: | ||
| with open(zip.extract(zip.namelist()[0]), 'r') as yaml_file: | ||
| return list(yaml.safe_load_all(yaml_file)) | ||
|
|
||
| def get_config(config_path): | ||
| with open(config_path) as file: | ||
| return list(yaml.safe_load_all(file)) | ||
|
|
||
| def get_params_from_config(pipeline_name, config_path): | ||
| pipelines = get_config(config_path) | ||
|
|
||
| for pipeline in pipelines: | ||
| if pipeline_name == pipeline["pipeline"]: | ||
| return pipeline | ||
|
|
||
| def test_workflow_without_decorator(pipeline_mod, params_dict): | ||
| """Test compiling a workflow and appending pipeline params.""" | ||
|
|
||
| try: | ||
| pipeline_params = [] | ||
| for param in params_dict.get('paramsList', []): | ||
| pipeline_params.append(getattr(pipeline_mod, param)) | ||
|
|
||
| compiled_workflow = compiler.TektonCompiler()._create_workflow( | ||
| getattr(pipeline_mod,params_dict['function']), | ||
| params_dict.get('name', None), | ||
| params_dict.get('description', None), | ||
| pipeline_params if pipeline_params else None, | ||
| params_dict.get('conf', None)) | ||
| return True | ||
| except : | ||
| return False | ||
|
|
||
| def test_nested_workflow(pipeline_mod, pipeline_list): | ||
| """Test compiling a simple workflow, and a bigger one composed from the simple one.""" | ||
|
|
||
| tmpdir = tempfile.mkdtemp() | ||
| try: | ||
| for pipeline in pipeline_list: | ||
| pipeline_name = pipeline['name'] | ||
| package_path = os.path.join(tmpdir, pipeline_name + '.zip') | ||
| compiler.TektonCompiler().compile(getattr(pipeline_mod, pipeline_name), package_path) | ||
| return True | ||
| except: | ||
| return False | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| test_data_path = sys.argv[1] | ||
| config_path = sys.argv[2] | ||
| did_compile = False | ||
|
|
||
| # Import pipeline | ||
| test_data_dir, test_data_file = os.path.split(test_data_path) | ||
| import_name, test_data_ext = os.path.splitext(test_data_file) | ||
| sys.path.append(test_data_dir) | ||
| pipeline_mod = importlib.import_module(import_name) | ||
|
|
||
| # Get the pipeline specific parameters from the config file | ||
| params = get_params_from_config(test_data_file, config_path) | ||
| if params == None: | ||
| raise ValueError('No pipeline matches available in the config file') | ||
| test_type = params['type'] | ||
|
|
||
| if test_type == 'nested': | ||
| did_compile = test_nested_workflow(pipeline_mod, params['components']) | ||
| elif test_type == 'no_decorator': | ||
| did_compile = test_workflow_without_decorator(pipeline_mod, params['components']) | ||
| else: | ||
| raise ValueError('Pipeline type \''+test_type+'\' is not recognized') | ||
| if did_compile: | ||
| print("SUCCESS:", test_data_file) | ||
| else: | ||
| print("FAILURE:", test_data_file) |