From 099ca6ad3bbeaf18a5af3a87e6f14de1e5b35d2f Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 23 Apr 2019 19:31:12 -0700 Subject: [PATCH 1/7] SDK - Separated the generated api client package --- sdk/python/kfp/_client.py | 45 +++++++++++++++------------------------ sdk/python/setup.py | 16 +------------- 2 files changed, 18 insertions(+), 43 deletions(-) diff --git a/sdk/python/kfp/_client.py b/sdk/python/kfp/_client.py index c70d946ca4e..13157bcf2db 100644 --- a/sdk/python/kfp/_client.py +++ b/sdk/python/kfp/_client.py @@ -47,14 +47,9 @@ def __init__(self, host=None, client_id=None): """ try: - import kfp_experiment + import kfp_server_api except ImportError: - raise Exception('This module requires installation of kfp_experiment') - - try: - import kfp_run - except ImportError: - raise Exception('This module requires installation of kfp_run') + raise Exception('This module requires the kfp-server-api package') self._host = host @@ -62,18 +57,12 @@ def __init__(self, host=None, client_id=None): if host and client_id: token = get_auth_token(client_id) - config = kfp_run.configuration.Configuration() - config.host = host if host else Client.IN_CLUSTER_DNS_NAME - self._configure_auth(config, token) - api_client = kfp_run.api_client.ApiClient(config) - self._run_api = kfp_run.api.run_service_api.RunServiceApi(api_client) - - config = kfp_experiment.configuration.Configuration() + config = kfp_server_api.configuration.Configuration() config.host = host if host else Client.IN_CLUSTER_DNS_NAME self._configure_auth(config, token) - api_client = kfp_experiment.api_client.ApiClient(config) - self._experiment_api = \ - kfp_experiment.api.experiment_service_api.ExperimentServiceApi(api_client) + api_client = kfp_server_api.api_client.ApiClient(config) + self._run_api = kfp_server_api.api.run_service_api.RunServiceApi(api_client) + self._experiment_api = kfp_server_api.api.experiment_service_api.ExperimentServiceApi(api_client) def _configure_auth(self, config, token): if token: @@ -110,7 +99,7 @@ def create_experiment(self, name): Returns: An Experiment object. Most important field is id. """ - import kfp_experiment + import kfp_server_api experiment = None try: @@ -121,7 +110,7 @@ def create_experiment(self, name): if not experiment: logging.info('Creating experiment {}.'.format(name)) - experiment = kfp_experiment.models.ApiExperiment(name=name) + experiment = kfp_server_api.models.ApiExperiment(name=name) experiment = self._experiment_api.create_experiment(body=experiment) if self._is_ipython(): @@ -213,22 +202,22 @@ def run_pipeline(self, experiment_id, job_name, pipeline_package_path=None, para Returns: A run object. Most important field is id. """ - import kfp_run + import kfp_server_api pipeline_json_string = None if pipeline_package_path: pipeline_obj = self._extract_pipeline_yaml(pipeline_package_path) pipeline_json_string = json.dumps(pipeline_obj) - api_params = [kfp_run.ApiParameter(name=_k8s_helper.K8sHelper.sanitize_k8s_name(k), value=str(v)) + api_params = [kfp_server_api.ApiParameter(name=_k8s_helper.K8sHelper.sanitize_k8s_name(k), value=str(v)) for k,v in params.items()] - key = kfp_run.models.ApiResourceKey(id=experiment_id, - type=kfp_run.models.ApiResourceType.EXPERIMENT) - reference = kfp_run.models.ApiResourceReference(key, kfp_run.models.ApiRelationship.OWNER) - spec = kfp_run.models.ApiPipelineSpec( + key = kfp_server_api.models.ApiResourceKey(id=experiment_id, + type=kfp_server_api.models.ApiResourceType.EXPERIMENT) + reference = kfp_server_api.models.ApiResourceReference(key, kfp_server_api.models.ApiRelationship.OWNER) + spec = kfp_server_api.models.ApiPipelineSpec( pipeline_id=pipeline_id, workflow_manifest=pipeline_json_string, parameters=api_params) - run_body = kfp_run.models.ApiRun( + run_body = kfp_server_api.models.ApiRun( pipeline_spec=spec, resource_references=[reference], name=job_name) response = self._run_api.create_run(body=run_body) @@ -251,8 +240,8 @@ def list_runs(self, page_token='', page_size=10, sort_by='', experiment_id=None) A response object including a list of experiments and next page token. """ if experiment_id is not None: - import kfp_run - response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by, resource_reference_key_type=kfp_run.models.api_resource_type.ApiResourceType.EXPERIMENT, resource_reference_key_id=experiment_id) + import kfp_server_api + response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by, resource_reference_key_type=kfp_server_api.models.api_resource_type.ApiResourceType.EXPERIMENT, resource_reference_key_id=experiment_id) else: response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by) return response diff --git a/sdk/python/setup.py b/sdk/python/setup.py index cff12e11a8d..e0fe6970d5a 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -29,6 +29,7 @@ 'cryptography>=2.4.2', 'google-auth>=1.6.1', 'requests_toolbelt>=0.8.0', + 'kfp-server-api>=0.1.18', ] setup( @@ -45,21 +46,6 @@ 'kfp.components.structures.kubernetes', 'kfp.dsl', 'kfp.notebook', - 'kfp_experiment', - 'kfp_experiment.api', - 'kfp_experiment.models', - 'kfp_run', - 'kfp_run.api', - 'kfp_run.models', - 'kfp_pipeline', - 'kfp_pipeline.api', - 'kfp_pipeline.models', - 'kfp_uploadpipeline', - 'kfp_uploadpipeline.api', - 'kfp_uploadpipeline.models', - 'kfp_job', - 'kfp_job.api', - 'kfp_job.models', ], classifiers=[ 'Intended Audience :: Developers', From f1f937f1857c6b14266b2219a3dbc4f06c6924a6 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 Apr 2019 18:16:32 -0700 Subject: [PATCH 2/7] Splitting the package build scripts --- sdk/python/build.sh | 43 +++----------- sdk/python/build_kfp_server_api_package.sh | 65 ++++++++++++++++++++++ 2 files changed, 72 insertions(+), 36 deletions(-) create mode 100755 sdk/python/build_kfp_server_api_package.sh diff --git a/sdk/python/build.sh b/sdk/python/build.sh index e901992e05e..f75b7b448b5 100755 --- a/sdk/python/build.sh +++ b/sdk/python/build.sh @@ -15,45 +15,16 @@ # limitations under the License. -# The scripts creates a Pipelines client python package. +# The scripts creates the Kubeflow Pipelines python SDK package. # # Usage: -# ./build.sh [output_dir] -# -# Setup: -# apt-get update -y -# apt-get install --no-install-recommends -y -q default-jdk -# wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.4.1/swagger-codegen-cli-2.4.1.jar -O /tmp/swagger-codegen-cli.jar +# ./build.sh [output_file] -get_abs_filename() { - # $1 : relative filename - echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" -} target_archive_file=${1:-kfp.tar.gz} -target_archive_file=$(get_abs_filename "$target_archive_file") - -DIR=$(mktemp -d) - -# Generate python code from swagger json. -echo "{\"packageName\": \"kfp_experiment\"}" > /tmp/config.json -java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/experiment.swagger.json -o $DIR -c /tmp/config.json -echo "{\"packageName\": \"kfp_run\"}" > /tmp/config.json -java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/run.swagger.json -o $DIR -c /tmp/config.json -echo "{\"packageName\": \"kfp_pipeline\"}" > /tmp/config.json -java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/pipeline.swagger.json -o $DIR -c /tmp/config.json -echo "{\"packageName\": \"kfp_uploadpipeline\"}" > /tmp/config.json -java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/pipeline.upload.swagger.json -o $DIR -c /tmp/config.json -echo "{\"packageName\": \"kfp_job\"}" > /tmp/config.json -java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/job.swagger.json -o $DIR -c /tmp/config.json -rm /tmp/config.json - -# Merge generated code with the rest code (setup.py, seira_client, etc). -cp -r kfp $DIR -cp ./setup.py $DIR -# Build tarball package. -cd $DIR -python setup.py sdist --format=gztar -cp $DIR/dist/*.tar.gz "$target_archive_file" -rm -rf $DIR +pushd "$(dirname "$0")" +dist_dir=$(mktemp -d) +python setup.py sdist --format=gztar --dist-dir "$dist_dir" +cp "$dist_dir"/*.tar.gz "$target_archive_file" +popd diff --git a/sdk/python/build_kfp_server_api_package.sh b/sdk/python/build_kfp_server_api_package.sh new file mode 100755 index 00000000000..f21453a434d --- /dev/null +++ b/sdk/python/build_kfp_server_api_package.sh @@ -0,0 +1,65 @@ +#!/bin/bash -e +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The scripts creates a the KF Pipelines API python package. +# Requirements: jq and Java +# sudo apt-get install --no-install-recommends -y -q default-jdk jq + +VERSION="$1" + +if [ -z "$VERSION" ]; then + echo "Usage: build_kfp_server_api_package.sh " + exit 1 +fi + +codegen_file=/tmp/swagger-codegen-cli.jar +codegen_uri=http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.4.1/swagger-codegen-cli-2.4.1.jar +if ! [ -f "$codegen_file" ]; then + wget "$codegen_uri" -O "$codegen_file" +fi + +pushd "$(dirname "$0")" + +DIR=$(mktemp -d) + +swagger_file=$(mktemp) + +echo "Merging all Swagger API definitions to $swagger_file." +jq -s ' + reduce .[] as $item ({}; . * $item) | + .info.title = "KF Pipelines API" | + .info.description = "Generated python client for the KF Pipelines server API" +' ../../backend/api/swagger/{run,job,pipeline,experiment,pipeline.upload}.swagger.json > "$swagger_file" + +echo "Generating python code from swagger json in $DIR." +java -jar "$codegen_file" generate -l python -i "$swagger_file" -o "$DIR" -c <(echo '{ + "packageName": "kfp_server_api", + "projectName": "kfp-server-api", + "packageVersion": "'"$VERSION"'", + "packageUrl": "https://github.com/kubeflow/pipelines" +}') + +echo "Building the python package in $DIR." +pushd "$DIR" +python3 setup.py --quiet sdist +popd + +echo "Run the following commands to update the package on PyPI" +echo "python3 -m pip install twine" +echo "python3 -m twine upload --username kubeflow-pipelines $DIR/dist/*" + +popd From dfca0f208ee5cdfa27ae97e44b1c653a4ef65988 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 Apr 2019 18:18:41 -0700 Subject: [PATCH 3/7] Pinning the API client package version --- sdk/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index e0fe6970d5a..f401329c9fe 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -29,7 +29,7 @@ 'cryptography>=2.4.2', 'google-auth>=1.6.1', 'requests_toolbelt>=0.8.0', - 'kfp-server-api>=0.1.18', + 'kfp-server-api==0.1.18.2', ] setup( From 23944655f1b8a24c0582b6043560383f46b7138c Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 Apr 2019 17:21:01 -0700 Subject: [PATCH 4/7] Moved import kfp_server_api to the top of the file --- sdk/python/kfp/_client.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sdk/python/kfp/_client.py b/sdk/python/kfp/_client.py index 13157bcf2db..8740c2a365b 100644 --- a/sdk/python/kfp/_client.py +++ b/sdk/python/kfp/_client.py @@ -22,6 +22,8 @@ import yaml from datetime import datetime +import kfp_server_api + from .compiler import compiler from .compiler import _k8s_helper @@ -46,11 +48,6 @@ def __init__(self, host=None, client_id=None): client_id: The client ID used by Identity-Aware Proxy. """ - try: - import kfp_server_api - except ImportError: - raise Exception('This module requires the kfp-server-api package') - self._host = host token = None @@ -99,7 +96,6 @@ def create_experiment(self, name): Returns: An Experiment object. Most important field is id. """ - import kfp_server_api experiment = None try: @@ -202,7 +198,6 @@ def run_pipeline(self, experiment_id, job_name, pipeline_package_path=None, para Returns: A run object. Most important field is id. """ - import kfp_server_api pipeline_json_string = None if pipeline_package_path: @@ -240,7 +235,6 @@ def list_runs(self, page_token='', page_size=10, sort_by='', experiment_id=None) A response object including a list of experiments and next page token. """ if experiment_id is not None: - import kfp_server_api response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by, resource_reference_key_type=kfp_server_api.models.api_resource_type.ApiResourceType.EXPERIMENT, resource_reference_key_id=experiment_id) else: response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by) From d9fa0122e52bdc318bd1615618a02a7f0862da44 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 Apr 2019 17:23:30 -0700 Subject: [PATCH 5/7] Added the Mac OS X prerequisite install instructions --- sdk/python/build_kfp_server_api_package.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk/python/build_kfp_server_api_package.sh b/sdk/python/build_kfp_server_api_package.sh index f21453a434d..464ff80dc00 100755 --- a/sdk/python/build_kfp_server_api_package.sh +++ b/sdk/python/build_kfp_server_api_package.sh @@ -17,7 +17,15 @@ # The scripts creates a the KF Pipelines API python package. # Requirements: jq and Java +# To install the prerequisites run the following: +# +# # Debian / Ubuntu: # sudo apt-get install --no-install-recommends -y -q default-jdk jq +# +# # OS X +# brew tap caskroom/cask +# brew cask install caskroom/versions/java8 +# brew install jq VERSION="$1" From 4abe0a507bd64eba19ce88dcfc593aecbc80694c Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 Apr 2019 17:30:18 -0700 Subject: [PATCH 6/7] Moved the build_kfp_server_api_python_package.sh script to the backend dir --- .../api/build_kfp_server_api_python_package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename sdk/python/build_kfp_server_api_package.sh => backend/api/build_kfp_server_api_python_package.sh (92%) diff --git a/sdk/python/build_kfp_server_api_package.sh b/backend/api/build_kfp_server_api_python_package.sh similarity index 92% rename from sdk/python/build_kfp_server_api_package.sh rename to backend/api/build_kfp_server_api_python_package.sh index 464ff80dc00..e31af27703f 100755 --- a/sdk/python/build_kfp_server_api_package.sh +++ b/backend/api/build_kfp_server_api_python_package.sh @@ -30,7 +30,7 @@ VERSION="$1" if [ -z "$VERSION" ]; then - echo "Usage: build_kfp_server_api_package.sh " + echo "Usage: build_kfp_server_api_python_package.sh " exit 1 fi @@ -51,7 +51,7 @@ jq -s ' reduce .[] as $item ({}; . * $item) | .info.title = "KF Pipelines API" | .info.description = "Generated python client for the KF Pipelines server API" -' ../../backend/api/swagger/{run,job,pipeline,experiment,pipeline.upload}.swagger.json > "$swagger_file" +' ./swagger/{run,job,pipeline,experiment,pipeline.upload}.swagger.json > "$swagger_file" echo "Generating python code from swagger json in $DIR." java -jar "$codegen_file" generate -l python -i "$swagger_file" -o "$DIR" -c <(echo '{ From 576b33bb9de62372ad0569ea849466aac92217a4 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 Apr 2019 11:41:29 -0700 Subject: [PATCH 7/7] Updated the dependency version span --- sdk/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index f401329c9fe..348559ee51e 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -29,7 +29,7 @@ 'cryptography>=2.4.2', 'google-auth>=1.6.1', 'requests_toolbelt>=0.8.0', - 'kfp-server-api==0.1.18.2', + 'kfp-server-api >= 0.1.18, < 0.1.19', #Update the upper version whenever a new version of the kfp-server-api package is released. Update the lower version when there is a breaking change in kfp-server-api. ] setup(