SDK - Separated the generated api client package

backend/api/build_kfp_server_api_python_package.sh

@@ -0,0 +1,73 @@
+#!/bin/bash -e
+#
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The scripts creates a the KF Pipelines API python package.
+# Requirements: jq and Java
+# To install the prerequisites run the following:
+#
+# # Debian / Ubuntu:
+# sudo apt-get install --no-install-recommends -y -q default-jdk jq
+#
+# # OS X
+# brew tap caskroom/cask
+# brew cask install caskroom/versions/java8
+# brew install jq
+
+VERSION="$1"
+
+if [ -z "$VERSION" ]; then
+    echo "Usage: build_kfp_server_api_python_package.sh <version>"
+    exit 1
+fi
+
+codegen_file=/tmp/swagger-codegen-cli.jar
+codegen_uri=http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.4.1/swagger-codegen-cli-2.4.1.jar
+if ! [ -f "$codegen_file" ]; then
+    wget "$codegen_uri" -O "$codegen_file"
+fi
+
+pushd "$(dirname "$0")"
+
+DIR=$(mktemp -d)
+
+swagger_file=$(mktemp)
+
+echo "Merging all Swagger API definitions to $swagger_file."
+jq -s '
+    reduce .[] as $item ({}; . * $item) |
+    .info.title = "KF Pipelines API" |
+    .info.description = "Generated python client for the KF Pipelines server API"
+' ./swagger/{run,job,pipeline,experiment,pipeline.upload}.swagger.json > "$swagger_file"
+
+echo "Generating python code from swagger json in $DIR."
+java -jar "$codegen_file" generate -l python -i "$swagger_file" -o "$DIR" -c <(echo '{
+    "packageName": "kfp_server_api",
+    "projectName": "kfp-server-api",
+    "packageVersion": "'"$VERSION"'",
+    "packageUrl": "https://github.com/kubeflow/pipelines"
+}')
+
+echo "Building the python package in $DIR."
+pushd "$DIR"
+python3 setup.py --quiet sdist
+popd
+
+echo "Run the following commands to update the package on PyPI"
+echo "python3 -m pip install twine"
+echo "python3 -m twine upload --username kubeflow-pipelines $DIR/dist/*"
+
+popd

sdk/python/build.sh

@@ -15,45 +15,16 @@
 # limitations under the License.
 
 
-# The scripts creates a Pipelines client python package.
+# The scripts creates the Kubeflow Pipelines python SDK package.
 #
 # Usage:
-#   ./build.sh [output_dir]
-#
-# Setup:
-#   apt-get update -y
-#   apt-get install --no-install-recommends -y -q default-jdk
-#   wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.4.1/swagger-codegen-cli-2.4.1.jar -O /tmp/swagger-codegen-cli.jar
+#   ./build.sh [output_file]
 
-get_abs_filename() {
-  # $1 : relative filename
-  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
-}
 
 target_archive_file=${1:-kfp.tar.gz}
-target_archive_file=$(get_abs_filename "$target_archive_file")
-
-DIR=$(mktemp -d)
-
-# Generate python code from swagger json.
-echo "{\"packageName\": \"kfp_experiment\"}" > /tmp/config.json
-java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/experiment.swagger.json -o $DIR -c /tmp/config.json
-echo "{\"packageName\": \"kfp_run\"}" > /tmp/config.json
-java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/run.swagger.json -o $DIR -c /tmp/config.json
-echo "{\"packageName\": \"kfp_pipeline\"}" > /tmp/config.json
-java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/pipeline.swagger.json -o $DIR -c /tmp/config.json
-echo "{\"packageName\": \"kfp_uploadpipeline\"}" > /tmp/config.json
-java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/pipeline.upload.swagger.json -o $DIR -c /tmp/config.json
-echo "{\"packageName\": \"kfp_job\"}" > /tmp/config.json
-java -jar /tmp/swagger-codegen-cli.jar generate -l python -i ../../backend/api/swagger/job.swagger.json -o $DIR -c /tmp/config.json
-rm /tmp/config.json
-
-# Merge generated code with the rest code (setup.py, seira_client, etc).
-cp -r kfp $DIR
-cp ./setup.py $DIR
 
-# Build tarball package.
-cd $DIR
-python setup.py sdist --format=gztar
-cp $DIR/dist/*.tar.gz "$target_archive_file"
-rm -rf $DIR
+pushd "$(dirname "$0")"
+dist_dir=$(mktemp -d)
+python setup.py sdist --format=gztar --dist-dir "$dist_dir"
+cp "$dist_dir"/*.tar.gz "$target_archive_file"
+popd

sdk/python/kfp/_client.py

@@ -22,6 +22,8 @@
 import yaml
 from datetime import datetime
 
+import kfp_server_api
+
 from .compiler import compiler
 from .compiler import _k8s_helper
 
@@ -46,34 +48,18 @@ def __init__(self, host=None, client_id=None):
       client_id: The client ID used by Identity-Aware Proxy.
     """
 
-    try:
-      import kfp_experiment
-    except ImportError:
-      raise Exception('This module requires installation of kfp_experiment')
-
-    try:
-      import kfp_run
-    except ImportError:
-      raise Exception('This module requires installation of kfp_run')
-
     self._host = host
 
     token = None
     if host and client_id:
       token = get_auth_token(client_id)
 
-    config = kfp_run.configuration.Configuration()
-    config.host = host if host else Client.IN_CLUSTER_DNS_NAME
-    self._configure_auth(config, token)
-    api_client = kfp_run.api_client.ApiClient(config)
-    self._run_api = kfp_run.api.run_service_api.RunServiceApi(api_client)
-
-    config = kfp_experiment.configuration.Configuration()
+    config = kfp_server_api.configuration.Configuration()
     config.host = host if host else Client.IN_CLUSTER_DNS_NAME
     self._configure_auth(config, token)
-    api_client = kfp_experiment.api_client.ApiClient(config)
-    self._experiment_api = \
-        kfp_experiment.api.experiment_service_api.ExperimentServiceApi(api_client)
+    api_client = kfp_server_api.api_client.ApiClient(config)
+    self._run_api = kfp_server_api.api.run_service_api.RunServiceApi(api_client)
+    self._experiment_api = kfp_server_api.api.experiment_service_api.ExperimentServiceApi(api_client)
 
   def _configure_auth(self, config, token):
     if token:
@@ -110,7 +96,6 @@ def create_experiment(self, name):
     Returns:
       An Experiment object. Most important field is id.
     """
-    import kfp_experiment
 
     experiment = None
     try:
@@ -121,7 +106,7 @@ def create_experiment(self, name):
 
     if not experiment:
       logging.info('Creating experiment {}.'.format(name))
-      experiment = kfp_experiment.models.ApiExperiment(name=name)
+      experiment = kfp_server_api.models.ApiExperiment(name=name)
       experiment = self._experiment_api.create_experiment(body=experiment)
 
     if self._is_ipython():
@@ -213,22 +198,21 @@ def run_pipeline(self, experiment_id, job_name, pipeline_package_path=None, para
     Returns:
       A run object. Most important field is id.
     """
-    import kfp_run
 
     pipeline_json_string = None
     if pipeline_package_path:
       pipeline_obj = self._extract_pipeline_yaml(pipeline_package_path)
       pipeline_json_string = json.dumps(pipeline_obj)
-    api_params = [kfp_run.ApiParameter(name=_k8s_helper.K8sHelper.sanitize_k8s_name(k), value=str(v))
+    api_params = [kfp_server_api.ApiParameter(name=_k8s_helper.K8sHelper.sanitize_k8s_name(k), value=str(v))
                   for k,v in params.items()]
-    key = kfp_run.models.ApiResourceKey(id=experiment_id,
-                                        type=kfp_run.models.ApiResourceType.EXPERIMENT)
-    reference = kfp_run.models.ApiResourceReference(key, kfp_run.models.ApiRelationship.OWNER)
-    spec = kfp_run.models.ApiPipelineSpec(
+    key = kfp_server_api.models.ApiResourceKey(id=experiment_id,
+                                        type=kfp_server_api.models.ApiResourceType.EXPERIMENT)
+    reference = kfp_server_api.models.ApiResourceReference(key, kfp_server_api.models.ApiRelationship.OWNER)
+    spec = kfp_server_api.models.ApiPipelineSpec(
         pipeline_id=pipeline_id,
         workflow_manifest=pipeline_json_string, 
         parameters=api_params)
-    run_body = kfp_run.models.ApiRun(
+    run_body = kfp_server_api.models.ApiRun(
         pipeline_spec=spec, resource_references=[reference], name=job_name)
 
     response = self._run_api.create_run(body=run_body)
@@ -251,8 +235,7 @@ def list_runs(self, page_token='', page_size=10, sort_by='', experiment_id=None)
       A response object including a list of experiments and next page token.
     """
     if experiment_id is not None:
-      import kfp_run
-      response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by, resource_reference_key_type=kfp_run.models.api_resource_type.ApiResourceType.EXPERIMENT, resource_reference_key_id=experiment_id)
+      response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by, resource_reference_key_type=kfp_server_api.models.api_resource_type.ApiResourceType.EXPERIMENT, resource_reference_key_id=experiment_id)
     else:
       response = self._run_api.list_runs(page_token=page_token, page_size=page_size, sort_by=sort_by)
     return response

sdk/python/setup.py

@@ -29,6 +29,7 @@
     'cryptography>=2.4.2',
     'google-auth>=1.6.1',
     'requests_toolbelt>=0.8.0',
+    'kfp-server-api >= 0.1.18, < 0.1.19', #Update the upper version whenever a new version of the kfp-server-api package is released. Update the lower version when there is a breaking change in kfp-server-api.
 ]
 
 setup(
@@ -45,21 +46,6 @@
         'kfp.components.structures.kubernetes',
         'kfp.dsl',
         'kfp.notebook',
-        'kfp_experiment',
-        'kfp_experiment.api',
-        'kfp_experiment.models',
-        'kfp_run',
-        'kfp_run.api',
-        'kfp_run.models',
-        'kfp_pipeline',
-        'kfp_pipeline.api',
-        'kfp_pipeline.models',
-        'kfp_uploadpipeline',
-        'kfp_uploadpipeline.api',
-        'kfp_uploadpipeline.models',
-        'kfp_job',
-        'kfp_job.api',
-        'kfp_job.models',
     ],
     classifiers=[
         'Intended Audience :: Developers',