From 46aafbd062b5ed231e3bc762c29810e9ff3f0119 Mon Sep 17 00:00:00 2001 From: Ning Date: Wed, 24 Jul 2019 14:24:08 -0700 Subject: [PATCH 01/14] move gcshelper out of component_builder (#1658) * move gcshelper out of component_builder --- sdk/python/kfp/compiler/_component_builder.py | 31 +------------ sdk/python/kfp/compiler/_gcs_helper.py | 44 +++++++++++++++++++ .../tests/compiler/component_builder_test.py | 1 - 3 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 sdk/python/kfp/compiler/_gcs_helper.py diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/compiler/_component_builder.py index 1f3d1f455e6a..435fd74a2deb 100644 --- a/sdk/python/kfp/compiler/_component_builder.py +++ b/sdk/python/kfp/compiler/_component_builder.py @@ -24,35 +24,6 @@ from pathlib import PurePath, Path from ..components._components import _create_task_factory_from_component_spec -class GCSHelper(object): - """ GCSHelper manages the connection with the GCS storage """ - - @staticmethod - def get_blob_from_gcs_uri(gcs_path): - from google.cloud import storage - pure_path = PurePath(gcs_path) - gcs_bucket = pure_path.parts[1] - gcs_blob = '/'.join(pure_path.parts[2:]) - client = storage.Client() - bucket = client.get_bucket(gcs_bucket) - blob = bucket.blob(gcs_blob) - return blob - - @staticmethod - def upload_gcs_file(local_path, gcs_path): - blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) - blob.upload_from_filename(local_path) - - @staticmethod - def remove_gcs_blob(gcs_path): - blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) - blob.delete() - - @staticmethod - def download_gcs_blob(local_path, gcs_path): - blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) - blob.download_to_filename(local_path) - class VersionedDependency(object): """ DependencyVersion specifies the versions """ def __init__(self, name, version=None, min_version=None, max_version=None): @@ -95,7 +66,6 @@ def has_max_version(self): def has_versions(self): return (self.has_min_version()) or (self.has_max_version()) - class DependencyHelper(object): """ DependencyHelper manages software dependency information """ def __init__(self): @@ -375,6 +345,7 @@ def _generate_entrypoint(self, component_func, python_version='python3'): return complete_component_code def _build_image_from_tarball(self, local_tarball_path, namespace, timeout): + from ._gcs_helper import GCSHelper GCSHelper.upload_gcs_file(local_tarball_path, self._gcs_path) kaniko_spec = self._generate_kaniko_spec(namespace=namespace, arc_dockerfile_name=self._arc_dockerfile_name, diff --git a/sdk/python/kfp/compiler/_gcs_helper.py b/sdk/python/kfp/compiler/_gcs_helper.py new file mode 100644 index 000000000000..3c8410713da5 --- /dev/null +++ b/sdk/python/kfp/compiler/_gcs_helper.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import PurePath + +class GCSHelper(object): + """ GCSHelper manages the connection with the GCS storage """ + + @staticmethod + def get_blob_from_gcs_uri(gcs_path): + from google.cloud import storage + pure_path = PurePath(gcs_path) + gcs_bucket = pure_path.parts[1] + gcs_blob = '/'.join(pure_path.parts[2:]) + client = storage.Client() + bucket = client.get_bucket(gcs_bucket) + blob = bucket.blob(gcs_blob) + return blob + + @staticmethod + def upload_gcs_file(local_path, gcs_path): + blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) + blob.upload_from_filename(local_path) + + @staticmethod + def remove_gcs_blob(gcs_path): + blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) + blob.delete() + + @staticmethod + def download_gcs_blob(local_path, gcs_path): + blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) + blob.download_to_filename(local_path) diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py index f45f09db645f..a3a70e129f0f 100644 --- a/sdk/python/tests/compiler/component_builder_test.py +++ b/sdk/python/tests/compiler/component_builder_test.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.compiler._component_builder import GCSHelper from kfp.compiler._component_builder import DockerfileHelper from kfp.compiler._component_builder import CodeGenerator from kfp.compiler._component_builder import ImageBuilder From 0eca44048c0189ff9191f6f187994a02e0784561 Mon Sep 17 00:00:00 2001 From: hongye-sun <43763191+hongye-sun@users.noreply.github.com> Date: Wed, 24 Jul 2019 15:51:51 -0700 Subject: [PATCH 02/14] Fix exit handler sample (#1665) --- samples/basic/exit_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/basic/exit_handler.py b/samples/basic/exit_handler.py index 25decd5d9e17..8adec022ff72 100755 --- a/samples/basic/exit_handler.py +++ b/samples/basic/exit_handler.py @@ -30,12 +30,13 @@ def gcs_download_op(url): ) -def echo_op(text): +def echo_op(text, is_exit_handler=False): return dsl.ContainerOp( name='echo', image='library/bash:4.4.23', command=['sh', '-c'], - arguments=['echo "$0"', text] + arguments=['echo "$0"', text], + is_exit_handler=is_exit_handler ) @@ -46,7 +47,7 @@ def echo_op(text): def download_and_print(url='gs://ml-pipeline-playground/shakespeare1.txt'): """A sample pipeline showing exit handler.""" - exit_task = echo_op('exit!') + exit_task = echo_op('exit!', is_exit_handler=True) with dsl.ExitHandler(exit_task): download_task = gcs_download_op(url) From 68cf384ad290100c35acf8ff4367182466fcf5de Mon Sep 17 00:00:00 2001 From: hongye-sun <43763191+hongye-sun@users.noreply.github.com> Date: Wed, 24 Jul 2019 17:31:51 -0700 Subject: [PATCH 03/14] Adding kfp.cil to kfp setup.py. (#1666) --- sdk/python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index ee5814143708..dac70636a4e9 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -45,6 +45,7 @@ install_requires=REQUIRES, packages=[ 'kfp', + 'kfp.cli', 'kfp.compiler', 'kfp.components', 'kfp.components.structures', From efff09cc1f70935ebd16f9319f0e4b58d0da28ac Mon Sep 17 00:00:00 2001 From: Kirin Patel Date: Thu, 25 Jul 2019 01:33:51 +0000 Subject: [PATCH 04/14] Add visualization server and unit tests for visualization server (#1647) * Added visualization server * Updated function names, added comments, and made serviceURL a property of VisualizationService * Added unit tests for visualizaiton.go * Updated BUILD.bazel * Addressed PR comments made by @IronPan * Wrapped input_path argument value in string when generating python arguments * GenerateVisualizationFromRequest -> generateVisualizationFromRequest * Addressed additional PR feedback from @IronPan * Removed getArgumentsAsJSONFromRequest * Removed createPythonArgumentsFromRequest * Moved `fmt.Sprintf` to generateVisualizationFromRequest * Updated tests to reflect changes * Added missing word to comment --- backend/src/apiserver/main.go | 2 + backend/src/apiserver/server/BUILD.bazel | 2 + .../apiserver/server/visualization_server.go | 78 ++++++++++++ .../server/visualization_server_test.go | 117 ++++++++++++++++++ 4 files changed, 199 insertions(+) create mode 100644 backend/src/apiserver/server/visualization_server.go create mode 100644 backend/src/apiserver/server/visualization_server_test.go diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 8d87bb47fc30..7bdd6243e571 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -83,6 +83,7 @@ func startRpcServer(resourceManager *resource.ResourceManager) { api.RegisterRunServiceServer(s, server.NewRunServer(resourceManager)) api.RegisterJobServiceServer(s, server.NewJobServer(resourceManager)) api.RegisterReportServiceServer(s, server.NewReportServer(resourceManager)) + api.RegisterVisualizationServiceServer(s, server.NewVisualizationServer(resourceManager)) // Register reflection service on gRPC server. reflection.Register(s) @@ -106,6 +107,7 @@ func startHttpProxy(resourceManager *resource.ResourceManager) { registerHttpHandlerFromEndpoint(api.RegisterJobServiceHandlerFromEndpoint, "JobService", ctx, mux) registerHttpHandlerFromEndpoint(api.RegisterRunServiceHandlerFromEndpoint, "RunService", ctx, mux) registerHttpHandlerFromEndpoint(api.RegisterReportServiceHandlerFromEndpoint, "ReportService", ctx, mux) + registerHttpHandlerFromEndpoint(api.RegisterVisualizationServiceHandlerFromEndpoint, "Visualization", ctx, mux) // Create a top level mux to include both pipeline upload server and gRPC servers. topMux := http.NewServeMux() diff --git a/backend/src/apiserver/server/BUILD.bazel b/backend/src/apiserver/server/BUILD.bazel index e72c46b11e16..83c307b0b8a4 100644 --- a/backend/src/apiserver/server/BUILD.bazel +++ b/backend/src/apiserver/server/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "run_server.go", "test_util.go", "util.go", + "visualization_server.go", ], importpath = "github.com/kubeflow/pipelines/backend/src/apiserver/server", visibility = ["//visibility:public"], @@ -50,6 +51,7 @@ go_test( "run_metric_util_test.go", "run_server_test.go", "util_test.go", + "visualization_server_test.go", ], data = glob(["test/**/*"]), # keep embed = [":go_default_library"], diff --git a/backend/src/apiserver/server/visualization_server.go b/backend/src/apiserver/server/visualization_server.go new file mode 100644 index 000000000000..e9982dbde57e --- /dev/null +++ b/backend/src/apiserver/server/visualization_server.go @@ -0,0 +1,78 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "github.com/kubeflow/pipelines/backend/api/go_client" + "github.com/kubeflow/pipelines/backend/src/apiserver/resource" + "github.com/kubeflow/pipelines/backend/src/common/util" + "io/ioutil" + "net/http" + "net/url" + "strings" +) + +type VisualizationServer struct { + resourceManager *resource.ResourceManager + serviceURL string +} + +func (s *VisualizationServer) CreateVisualization(ctx context.Context, request *go_client.CreateVisualizationRequest) (*go_client.Visualization, error) { + if err := s.validateCreateVisualizationRequest(request); err != nil { + return nil, err + } + body, err := s.generateVisualizationFromRequest(request) + if err != nil { + return nil, err + } + request.Visualization.Html = string(body) + return request.Visualization, nil +} + +// validateCreateVisualizationRequest ensures that a go_client.Visualization +// object has valid values. +// It returns an error if a go_client.Visualization object does not have valid +// values. +func (s *VisualizationServer) validateCreateVisualizationRequest(request *go_client.CreateVisualizationRequest) error { + if len(request.Visualization.InputPath) == 0 { + return util.NewInvalidInputError("A visualization requires an InputPath to be provided. Received %s", request.Visualization.InputPath) + } + // Manually set Arguments to empty JSON if nothing is provided. This is done + // because visualizations such as TFDV and TFMA only require an InputPath to + // be provided for a visualization to be generated. If no JSON is provided + // json.Valid will fail without this check as an empty string is provided for + // those visualizations. + if len(request.Visualization.Arguments) == 0 { + request.Visualization.Arguments = "{}" + } + if !json.Valid([]byte(request.Visualization.Arguments)) { + return util.NewInvalidInputError("A visualization requires valid JSON to be provided as Arguments. Received %s", request.Visualization.Arguments) + } + return nil +} + +// generateVisualizationFromRequest communicates with the python visualization +// service to generate HTML visualizations from a request. +// It returns the generated HTML as a string and any error that is encountered. +func (s *VisualizationServer) generateVisualizationFromRequest(request *go_client.CreateVisualizationRequest) ([]byte, error) { + visualizationType := strings.ToLower(go_client.Visualization_Type_name[int32(request.Visualization.Type)]) + arguments := fmt.Sprintf("--type %s --input_path %s --arguments '%s'", visualizationType, request.Visualization.InputPath, request.Visualization.Arguments) + resp, err := http.PostForm(s.serviceURL, url.Values{"arguments": {arguments}}) + if err != nil { + return nil, util.Wrap(err, "Unable to initialize visualization request.") + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf(resp.Status) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, util.Wrap(err, "Unable to parse visualization response.") + } + return body, nil +} + +func NewVisualizationServer(resourceManager *resource.ResourceManager) *VisualizationServer { + return &VisualizationServer{resourceManager: resourceManager, serviceURL: "http://visualization-service.kubeflow"} +} diff --git a/backend/src/apiserver/server/visualization_server_test.go b/backend/src/apiserver/server/visualization_server_test.go new file mode 100644 index 000000000000..bceb52785fef --- /dev/null +++ b/backend/src/apiserver/server/visualization_server_test.go @@ -0,0 +1,117 @@ +package server + +import ( + "github.com/kubeflow/pipelines/backend/api/go_client" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +func TestValidateCreateVisualizationRequest(t *testing.T) { + clients, manager, _ := initWithExperiment(t) + defer clients.Close() + server := NewVisualizationServer(manager) + visualization := &go_client.Visualization{ + Type: go_client.Visualization_ROC_CURVE, + InputPath: "gs://ml-pipeline/roc/data.csv", + Arguments: "{}", + } + request := &go_client.CreateVisualizationRequest{ + Visualization: visualization, + } + err := server.validateCreateVisualizationRequest(request) + assert.Nil(t, err) +} + +func TestValidateCreateVisualizationRequest_ArgumentsAreEmpty(t *testing.T) { + clients, manager, _ := initWithExperiment(t) + defer clients.Close() + server := NewVisualizationServer(manager) + visualization := &go_client.Visualization{ + Type: go_client.Visualization_ROC_CURVE, + InputPath: "gs://ml-pipeline/roc/data.csv", + Arguments: "", + } + request := &go_client.CreateVisualizationRequest{ + Visualization: visualization, + } + err := server.validateCreateVisualizationRequest(request) + assert.Nil(t, err) +} + +func TestValidateCreateVisualizationRequest_InputPathIsEmpty(t *testing.T) { + clients, manager, _ := initWithExperiment(t) + defer clients.Close() + server := NewVisualizationServer(manager) + visualization := &go_client.Visualization{ + Type: go_client.Visualization_ROC_CURVE, + InputPath: "", + Arguments: "{}", + } + request := &go_client.CreateVisualizationRequest{ + Visualization: visualization, + } + err := server.validateCreateVisualizationRequest(request) + assert.Contains(t, err.Error(), "A visualization requires an InputPath to be provided. Received") +} + +func TestValidateCreateVisualizationRequest_ArgumentsNotValidJSON(t *testing.T) { + clients, manager, _ := initWithExperiment(t) + defer clients.Close() + server := NewVisualizationServer(manager) + visualization := &go_client.Visualization{ + Type: go_client.Visualization_ROC_CURVE, + InputPath: "gs://ml-pipeline/roc/data.csv", + Arguments: "{", + } + request := &go_client.CreateVisualizationRequest{ + Visualization: visualization, + } + err := server.validateCreateVisualizationRequest(request) + assert.Contains(t, err.Error(), "A visualization requires valid JSON to be provided as Arguments. Received {") +} + +func TestGenerateVisualization(t *testing.T) { + clients, manager, _ := initWithExperiment(t) + defer clients.Close() + httpServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + assert.Equal(t, "/", req.URL.String()) + rw.Write([]byte("roc_curve")) + })) + defer httpServer.Close() + server := &VisualizationServer{resourceManager: manager, serviceURL: httpServer.URL} + visualization := &go_client.Visualization{ + Type: go_client.Visualization_ROC_CURVE, + InputPath: "gs://ml-pipeline/roc/data.csv", + Arguments: "{}", + } + request := &go_client.CreateVisualizationRequest{ + Visualization: visualization, + } + body, err := server.generateVisualizationFromRequest(request) + assert.Equal(t, []byte("roc_curve"), body) + assert.Nil(t, err) +} + +func TestGenerateVisualization_ServerError(t *testing.T) { + clients, manager, _ := initWithExperiment(t) + defer clients.Close() + httpServer := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + assert.Equal(t, "/", req.URL.String()) + rw.WriteHeader(500) + })) + defer httpServer.Close() + server := &VisualizationServer{resourceManager: manager, serviceURL: httpServer.URL} + visualization := &go_client.Visualization{ + Type: go_client.Visualization_ROC_CURVE, + InputPath: "gs://ml-pipeline/roc/data.csv", + Arguments: "{}", + } + request := &go_client.CreateVisualizationRequest{ + Visualization: visualization, + } + body, err := server.generateVisualizationFromRequest(request) + assert.Nil(t, body) + assert.Equal(t, "500 Internal Server Error", err.Error()) +} From 82391f4d1dca5636a79faab4bc77ef640f1134ed Mon Sep 17 00:00:00 2001 From: Kirin Patel Date: Thu, 25 Jul 2019 02:39:51 +0000 Subject: [PATCH 05/14] Add visualization swagger files to frontend (#1663) * Generated visualization swagger files for frontend * Changed swagger codegen cli version from 2.4.1 to 2.3.1 Regenerated visualization API with swagger codegen cli version 2.3.1 --- frontend/package.json | 3 +- frontend/src/apis/visualization/.gitignore | 3 + .../visualization/.swagger-codegen-ignore | 23 ++ .../visualization/.swagger-codegen/VERSION | 1 + frontend/src/apis/visualization/api.ts | 287 ++++++++++++++++++ .../src/apis/visualization/configuration.ts | 66 ++++ frontend/src/apis/visualization/custom.d.ts | 1 + frontend/src/apis/visualization/git_push.sh | 51 ++++ frontend/src/apis/visualization/index.ts | 16 + 9 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 frontend/src/apis/visualization/.gitignore create mode 100644 frontend/src/apis/visualization/.swagger-codegen-ignore create mode 100644 frontend/src/apis/visualization/.swagger-codegen/VERSION create mode 100644 frontend/src/apis/visualization/api.ts create mode 100644 frontend/src/apis/visualization/configuration.ts create mode 100644 frontend/src/apis/visualization/custom.d.ts create mode 100644 frontend/src/apis/visualization/git_push.sh create mode 100644 frontend/src/apis/visualization/index.ts diff --git a/frontend/package.json b/frontend/package.json index 46ff947b7e3f..ee01ec5ea47a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -29,12 +29,13 @@ }, "scripts": { "analyze-bundle": "node analyze_bundle.js", - "apis": "npm run apis:experiment && npm run apis:job && npm run apis:pipeline && npm run apis:run && npm run apis:filter", + "apis": "npm run apis:experiment && npm run apis:job && npm run apis:pipeline && npm run apis:run && npm run apis:filter && npm run apis:visualization", "apis:experiment": "java -jar swagger-codegen-cli.jar generate -i ../backend/api/swagger/experiment.swagger.json -l typescript-fetch -o ./src/apis/experiment -c ./swagger-config.json", "apis:job": "java -jar swagger-codegen-cli.jar generate -i ../backend/api/swagger/job.swagger.json -l typescript-fetch -o ./src/apis/job -c ./swagger-config.json", "apis:pipeline": "java -jar swagger-codegen-cli.jar generate -i ../backend/api/swagger/pipeline.swagger.json -l typescript-fetch -o ./src/apis/pipeline -c ./swagger-config.json", "apis:run": "java -jar swagger-codegen-cli.jar generate -i ../backend/api/swagger/run.swagger.json -l typescript-fetch -o ./src/apis/run -c ./swagger-config.json", "apis:filter": "java -jar swagger-codegen-cli.jar generate -i ../backend/api/swagger/filter.swagger.json -l typescript-fetch -o ./src/apis/filter -c ./swagger-config.json", + "apis:visualization": "java -jar swagger-codegen-cli.jar generate -i ../backend/api/swagger/visualization.swagger.json -l typescript-fetch -o ./src/apis/visualization -c ./swagger-config.json", "build": "react-scripts-ts build", "docker": "COMMIT_HASH=`git rev-parse HEAD`; docker build -q -t ml-pipelines-frontend:${COMMIT_HASH} --build-arg COMMIT_HASH=${COMMIT_HASH} --build-arg DATE=\"`date -u`\" -f Dockerfile ..", "eject": "react-scripts-ts eject", diff --git a/frontend/src/apis/visualization/.gitignore b/frontend/src/apis/visualization/.gitignore new file mode 100644 index 000000000000..35e2fb2b02ed --- /dev/null +++ b/frontend/src/apis/visualization/.gitignore @@ -0,0 +1,3 @@ +wwwroot/*.js +node_modules +typings diff --git a/frontend/src/apis/visualization/.swagger-codegen-ignore b/frontend/src/apis/visualization/.swagger-codegen-ignore new file mode 100644 index 000000000000..c5fa491b4c55 --- /dev/null +++ b/frontend/src/apis/visualization/.swagger-codegen-ignore @@ -0,0 +1,23 @@ +# Swagger Codegen Ignore +# Generated by swagger-codegen https://github.com/swagger-api/swagger-codegen + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell Swagger Codgen to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/frontend/src/apis/visualization/.swagger-codegen/VERSION b/frontend/src/apis/visualization/.swagger-codegen/VERSION new file mode 100644 index 000000000000..a6254504e401 --- /dev/null +++ b/frontend/src/apis/visualization/.swagger-codegen/VERSION @@ -0,0 +1 @@ +2.3.1 \ No newline at end of file diff --git a/frontend/src/apis/visualization/api.ts b/frontend/src/apis/visualization/api.ts new file mode 100644 index 000000000000..4e712dc660c1 --- /dev/null +++ b/frontend/src/apis/visualization/api.ts @@ -0,0 +1,287 @@ +/// +// tslint:disable +/** + * backend/api/visualization.proto + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * OpenAPI spec version: version not set + * + * + * NOTE: This class is auto generated by the swagger code generator program. + * https://github.com/swagger-api/swagger-codegen.git + * Do not edit the class manually. + */ + + +import * as url from "url"; +import * as portableFetch from "portable-fetch"; +import { Configuration } from "./configuration"; + +const BASE_PATH = "http://localhost".replace(/\/+$/, ""); + +/** + * + * @export + */ +export const COLLECTION_FORMATS = { + csv: ",", + ssv: " ", + tsv: "\t", + pipes: "|", +}; + +/** + * + * @export + * @interface FetchAPI + */ +export interface FetchAPI { + (url: string, init?: any): Promise; +} + +/** + * + * @export + * @interface FetchArgs + */ +export interface FetchArgs { + url: string; + options: any; +} + +/** + * + * @export + * @class BaseAPI + */ +export class BaseAPI { + protected configuration: Configuration; + + constructor(configuration?: Configuration, protected basePath: string = BASE_PATH, protected fetch: FetchAPI = portableFetch) { + if (configuration) { + this.configuration = configuration; + this.basePath = configuration.basePath || this.basePath; + } + } +}; + +/** + * + * @export + * @class RequiredError + * @extends {Error} + */ +export class RequiredError extends Error { + name: "RequiredError" + constructor(public field: string, msg?: string) { + super(msg); + } +} + +/** + * + * @export + * @interface ApiStatus + */ +export interface ApiStatus { + /** + * + * @type {string} + * @memberof ApiStatus + */ + error?: string; + /** + * + * @type {number} + * @memberof ApiStatus + */ + code?: number; + /** + * + * @type {Array<ProtobufAny>} + * @memberof ApiStatus + */ + details?: Array; +} + +/** + * + * @export + * @interface ApiVisualization + */ +export interface ApiVisualization { + /** + * + * @type {ApiVisualizationType} + * @memberof ApiVisualization + */ + type?: ApiVisualizationType; + /** + * Path pattern of input data to be used during generation of visualizations. This is required when creating the pipeline through CreateVisualization API. + * @type {string} + * @memberof ApiVisualization + */ + inputPath?: string; + /** + * Variables to be used during generation of a visualization. This should be provided as a JSON string. This is required when creating the pipeline through CreateVisualization API. + * @type {string} + * @memberof ApiVisualization + */ + arguments?: string; + /** + * Output. Generated visualization html. + * @type {string} + * @memberof ApiVisualization + */ + html?: string; + /** + * In case any error happens when generating visualizations, only visualization ID and the error message are returned. Client has the flexibility of choosing how to handle the error. + * @type {string} + * @memberof ApiVisualization + */ + error?: string; +} + +/** + * Type of visualization to be generated. This is required when creating the pipeline through CreateVisualization API. + * @export + * @enum {string} + */ +export enum ApiVisualizationType { + CURVE = 'ROC_CURVE' +} + +/** + * `Any` contains an arbitrary serialized protocol buffer message along with a URL that describes the type of the serialized message. Protobuf library provides support to pack/unpack Any values in the form of utility functions or additional generated methods of the Any type. Example 1: Pack and unpack a message in C++. Foo foo = ...; Any any; any.PackFrom(foo); ... if (any.UnpackTo(&foo)) { ... } Example 2: Pack and unpack a message in Java. Foo foo = ...; Any any = Any.pack(foo); ... if (any.is(Foo.class)) { foo = any.unpack(Foo.class); } Example 3: Pack and unpack a message in Python. foo = Foo(...) any = Any() any.Pack(foo) ... if any.Is(Foo.DESCRIPTOR): any.Unpack(foo) ... Example 4: Pack and unpack a message in Go foo := &pb.Foo{...} any, err := ptypes.MarshalAny(foo) ... foo := &pb.Foo{} if err := ptypes.UnmarshalAny(any, foo); err != nil { ... } The pack methods provided by protobuf library will by default use 'type.googleapis.com/full.type.name' as the type URL and the unpack methods only use the fully qualified type name after the last '/' in the type URL, for example \"foo.bar.com/x/y.z\" will yield type name \"y.z\". JSON ==== The JSON representation of an `Any` value uses the regular representation of the deserialized, embedded message, with an additional field `@type` which contains the type URL. Example: package google.profile; message Person { string first_name = 1; string last_name = 2; } { \"@type\": \"type.googleapis.com/google.profile.Person\", \"firstName\": , \"lastName\": } If the embedded message type is well-known and has a custom JSON representation, that representation will be embedded adding a field `value` which holds the custom JSON in addition to the `@type` field. Example (for message [google.protobuf.Duration][]): { \"@type\": \"type.googleapis.com/google.protobuf.Duration\", \"value\": \"1.212s\" } + * @export + * @interface ProtobufAny + */ +export interface ProtobufAny { + /** + * A URL/resource name that uniquely identifies the type of the serialized protocol buffer message. The last segment of the URL's path must represent the fully qualified name of the type (as in `path/google.protobuf.Duration`). The name should be in a canonical form (e.g., leading \".\" is not accepted). In practice, teams usually precompile into the binary all types that they expect it to use in the context of Any. However, for URLs which use the scheme `http`, `https`, or no scheme, one can optionally set up a type server that maps type URLs to message definitions as follows: * If no scheme is provided, `https` is assumed. * An HTTP GET on the URL must yield a [google.protobuf.Type][] value in binary format, or produce an error. * Applications are allowed to cache lookup results based on the URL, or have them precompiled into a binary to avoid any lookup. Therefore, binary compatibility needs to be preserved on changes to types. (Use versioned type names to manage breaking changes.) Note: this functionality is not currently available in the official protobuf release, and it is not used for type URLs beginning with type.googleapis.com. Schemes other than `http`, `https` (or the empty scheme) might be used with implementation specific semantics. + * @type {string} + * @memberof ProtobufAny + */ + type_url?: string; + /** + * Must be a valid serialized protocol buffer of the above specified type. + * @type {string} + * @memberof ProtobufAny + */ + value?: string; +} + + +/** + * VisualizationServiceApi - fetch parameter creator + * @export + */ +export const VisualizationServiceApiFetchParamCreator = function (configuration?: Configuration) { + return { + /** + * + * @param {ApiVisualization} body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + createVisualization(body: ApiVisualization, options: any = {}): FetchArgs { + // verify required parameter 'body' is not null or undefined + if (body === null || body === undefined) { + throw new RequiredError('body','Required parameter body was null or undefined when calling createVisualization.'); + } + const localVarPath = `/apis/v1beta1/visualizations`; + const localVarUrlObj = url.parse(localVarPath, true); + const localVarRequestOptions = Object.assign({ method: 'POST' }, options); + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication Bearer required + if (configuration && configuration.apiKey) { + const localVarApiKeyValue = typeof configuration.apiKey === 'function' + ? configuration.apiKey("authorization") + : configuration.apiKey; + localVarHeaderParameter["authorization"] = localVarApiKeyValue; + } + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); + // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 + delete localVarUrlObj.search; + localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); + const needsSerialization = ("ApiVisualization" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; + localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); + + return { + url: url.format(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * VisualizationServiceApi - functional programming interface + * @export + */ +export const VisualizationServiceApiFp = function(configuration?: Configuration) { + return { + /** + * + * @param {ApiVisualization} body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + createVisualization(body: ApiVisualization, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { + const localVarFetchArgs = VisualizationServiceApiFetchParamCreator(configuration).createVisualization(body, options); + return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { + if (response.status >= 200 && response.status < 300) { + return response.json(); + } else { + throw response; + } + }); + }; + }, + } +}; + +/** + * VisualizationServiceApi - factory interface + * @export + */ +export const VisualizationServiceApiFactory = function (configuration?: Configuration, fetch?: FetchAPI, basePath?: string) { + return { + /** + * + * @param {ApiVisualization} body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + createVisualization(body: ApiVisualization, options?: any) { + return VisualizationServiceApiFp(configuration).createVisualization(body, options)(fetch, basePath); + }, + }; +}; + +/** + * VisualizationServiceApi - object-oriented interface + * @export + * @class VisualizationServiceApi + * @extends {BaseAPI} + */ +export class VisualizationServiceApi extends BaseAPI { + /** + * + * @param {} body + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof VisualizationServiceApi + */ + public createVisualization(body: ApiVisualization, options?: any) { + return VisualizationServiceApiFp(this.configuration).createVisualization(body, options)(this.fetch, this.basePath); + } + +} + diff --git a/frontend/src/apis/visualization/configuration.ts b/frontend/src/apis/visualization/configuration.ts new file mode 100644 index 000000000000..d81fb1744b9e --- /dev/null +++ b/frontend/src/apis/visualization/configuration.ts @@ -0,0 +1,66 @@ +// tslint:disable +/** + * backend/api/visualization.proto + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * OpenAPI spec version: version not set + * + * + * NOTE: This class is auto generated by the swagger code generator program. + * https://github.com/swagger-api/swagger-codegen.git + * Do not edit the class manually. + */ + + +export interface ConfigurationParameters { + apiKey?: string | ((name: string) => string); + username?: string; + password?: string; + accessToken?: string | ((name: string, scopes?: string[]) => string); + basePath?: string; +} + +export class Configuration { + /** + * parameter for apiKey security + * @param name security name + * @memberof Configuration + */ + apiKey?: string | ((name: string) => string); + /** + * parameter for basic security + * + * @type {string} + * @memberof Configuration + */ + username?: string; + /** + * parameter for basic security + * + * @type {string} + * @memberof Configuration + */ + password?: string; + /** + * parameter for oauth2 security + * @param name security name + * @param scopes oauth2 scope + * @memberof Configuration + */ + accessToken?: string | ((name: string, scopes?: string[]) => string); + /** + * override base path + * + * @type {string} + * @memberof Configuration + */ + basePath?: string; + + constructor(param: ConfigurationParameters = {}) { + this.apiKey = param.apiKey; + this.username = param.username; + this.password = param.password; + this.accessToken = param.accessToken; + this.basePath = param.basePath; + } +} diff --git a/frontend/src/apis/visualization/custom.d.ts b/frontend/src/apis/visualization/custom.d.ts new file mode 100644 index 000000000000..02f969575e37 --- /dev/null +++ b/frontend/src/apis/visualization/custom.d.ts @@ -0,0 +1 @@ +declare module 'portable-fetch'; \ No newline at end of file diff --git a/frontend/src/apis/visualization/git_push.sh b/frontend/src/apis/visualization/git_push.sh new file mode 100644 index 000000000000..a1ff4c9bcba8 --- /dev/null +++ b/frontend/src/apis/visualization/git_push.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/ +# +# Usage example: /bin/sh ./git_push.sh wing328 swagger-petstore-perl "minor update" + +git_user_id=$1 +git_repo_id=$2 +release_note=$3 + +if [ "$git_user_id" = "" ]; then + git_user_id="GIT_USER_ID" + echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id" +fi + +if [ "$git_repo_id" = "" ]; then + git_repo_id="GIT_REPO_ID" + echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id" +fi + +if [ "$release_note" = "" ]; then + release_note="Minor update" + echo "[INFO] No command line input provided. Set \$release_note to $release_note" +fi + +# Initialize the local directory as a Git repository +git init + +# Adds the files in the local repository and stages them for commit. +git add . + +# Commits the tracked changes and prepares them to be pushed to a remote repository. +git commit -m "$release_note" + +# Sets the new remote +git_remote=`git remote` +if [ "$git_remote" = "" ]; then # git remote not defined + + if [ "$GIT_TOKEN" = "" ]; then + echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment." + git remote add origin https://github.com/${git_user_id}/${git_repo_id}.git + else + git remote add origin https://${git_user_id}:${GIT_TOKEN}@github.com/${git_user_id}/${git_repo_id}.git + fi + +fi + +git pull origin master + +# Pushes (Forces) the changes in the local repository up to the remote repository +echo "Git pushing to https://github.com/${git_user_id}/${git_repo_id}.git" +git push origin master 2>&1 | grep -v 'To https' diff --git a/frontend/src/apis/visualization/index.ts b/frontend/src/apis/visualization/index.ts new file mode 100644 index 000000000000..349871961dd4 --- /dev/null +++ b/frontend/src/apis/visualization/index.ts @@ -0,0 +1,16 @@ +// tslint:disable +/** + * backend/api/visualization.proto + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * OpenAPI spec version: version not set + * + * + * NOTE: This class is auto generated by the swagger code generator program. + * https://github.com/swagger-api/swagger-codegen.git + * Do not edit the class manually. + */ + + +export * from "./api"; +export * from "./configuration"; From 2355451885ba017a4186db9ac520331ee2dbcb94 Mon Sep 17 00:00:00 2001 From: jingzhang36 Date: Thu, 25 Jul 2019 12:05:52 +0800 Subject: [PATCH 06/14] Release fe639f41661d8e17fcda64ff8242127620b80ba0 (#1672) * Updated component images to version fe639f41661d8e17fcda64ff8242127620b80ba0 * Updated components to version 0b07e456b1f319d8b7a7301274f55c00fda9f537 --- components/dataflow/predict/component.yaml | 2 +- components/dataflow/tfdv/component.yaml | 2 +- components/dataflow/tfma/component.yaml | 2 +- components/dataflow/tft/component.yaml | 2 +- components/gcp/bigquery/query/README.md | 2 +- components/gcp/bigquery/query/component.yaml | 2 +- components/gcp/bigquery/query/sample.ipynb | 2 +- components/gcp/dataflow/launch_python/README.md | 2 +- .../gcp/dataflow/launch_python/component.yaml | 2 +- .../gcp/dataflow/launch_python/sample.ipynb | 2 +- .../gcp/dataflow/launch_template/README.md | 2 +- .../gcp/dataflow/launch_template/component.yaml | 2 +- .../gcp/dataflow/launch_template/sample.ipynb | 2 +- components/gcp/dataproc/create_cluster/README.md | 2 +- .../gcp/dataproc/create_cluster/component.yaml | 2 +- .../gcp/dataproc/create_cluster/sample.ipynb | 2 +- components/gcp/dataproc/delete_cluster/README.md | 2 +- .../gcp/dataproc/delete_cluster/component.yaml | 2 +- .../gcp/dataproc/delete_cluster/sample.ipynb | 2 +- .../gcp/dataproc/submit_hadoop_job/README.md | 2 +- .../dataproc/submit_hadoop_job/component.yaml | 2 +- .../gcp/dataproc/submit_hadoop_job/sample.ipynb | 2 +- .../gcp/dataproc/submit_hive_job/README.md | 2 +- .../gcp/dataproc/submit_hive_job/component.yaml | 2 +- .../gcp/dataproc/submit_hive_job/sample.ipynb | 2 +- components/gcp/dataproc/submit_pig_job/README.md | 2 +- .../gcp/dataproc/submit_pig_job/component.yaml | 2 +- .../gcp/dataproc/submit_pig_job/sample.ipynb | 2 +- .../gcp/dataproc/submit_pyspark_job/README.md | 2 +- .../dataproc/submit_pyspark_job/component.yaml | 2 +- .../gcp/dataproc/submit_pyspark_job/sample.ipynb | 2 +- .../gcp/dataproc/submit_spark_job/README.md | 2 +- .../gcp/dataproc/submit_spark_job/component.yaml | 2 +- .../gcp/dataproc/submit_spark_job/sample.ipynb | 2 +- .../gcp/dataproc/submit_sparksql_job/README.md | 2 +- .../dataproc/submit_sparksql_job/component.yaml | 2 +- .../dataproc/submit_sparksql_job/sample.ipynb | 2 +- components/gcp/ml_engine/batch_predict/README.md | 2 +- .../gcp/ml_engine/batch_predict/component.yaml | 2 +- .../gcp/ml_engine/batch_predict/sample.ipynb | 2 +- components/gcp/ml_engine/deploy/README.md | 2 +- components/gcp/ml_engine/deploy/component.yaml | 2 +- components/gcp/ml_engine/deploy/sample.ipynb | 2 +- components/gcp/ml_engine/train/README.md | 2 +- components/gcp/ml_engine/train/component.yaml | 2 +- components/gcp/ml_engine/train/sample.ipynb | 2 +- components/kubeflow/deployer/component.yaml | 2 +- components/kubeflow/dnntrainer/component.yaml | 2 +- .../launcher/kubeflow_tfjob_launcher_op.py | 2 +- .../kubeflow/launcher/src/train.template.yaml | 6 +++--- components/local/confusion_matrix/component.yaml | 2 +- components/local/roc/component.yaml | 2 +- samples/ai-platform/Chicago Crime Pipeline.ipynb | 6 +++--- .../kubeflow-training-classification.py | 10 +++++----- ...eFlow Pipeline Using TFX OSS Components.ipynb | 14 +++++++------- samples/resnet-cmle/resnet-train-pipeline.py | 6 +++--- samples/tfx/taxi-cab-classification-pipeline.py | 16 ++++++++-------- samples/xgboost-spark/xgboost-training-cm.py | 16 ++++++++-------- 58 files changed, 88 insertions(+), 88 deletions(-) diff --git a/components/dataflow/predict/component.yaml b/components/dataflow/predict/component.yaml index fe2da7dfb76a..846e085cc1b8 100644 --- a/components/dataflow/predict/component.yaml +++ b/components/dataflow/predict/component.yaml @@ -15,7 +15,7 @@ outputs: - {name: Predictions dir, type: GCSPath, description: 'GCS or local directory.'} #Will contain prediction_results-* and schema.json files; TODO: Split outputs and replace dir with single file # type: {GCSPath: {path_type: Directory}} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, /ml/predict.py] args: [ --data, {inputValue: Data file pattern}, diff --git a/components/dataflow/tfdv/component.yaml b/components/dataflow/tfdv/component.yaml index 394e2c54b843..009ebd3a03b4 100644 --- a/components/dataflow/tfdv/component.yaml +++ b/components/dataflow/tfdv/component.yaml @@ -18,7 +18,7 @@ outputs: - {name: Validation result, type: String, description: Indicates whether anomalies were detected or not.} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, /ml/validate.py] args: [ --csv-data-for-inference, {inputValue: Inference data}, diff --git a/components/dataflow/tfma/component.yaml b/components/dataflow/tfma/component.yaml index 56885faa2dd5..940b72bee6dd 100644 --- a/components/dataflow/tfma/component.yaml +++ b/components/dataflow/tfma/component.yaml @@ -17,7 +17,7 @@ outputs: - {name: Analysis results dir, type: GCSPath, description: GCS or local directory where the analysis results should were written.} # type: {GCSPath: {path_type: Directory}} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, /ml/model_analysis.py] args: [ --model, {inputValue: Model}, diff --git a/components/dataflow/tft/component.yaml b/components/dataflow/tft/component.yaml index 7c98a41fa556..f004defee54c 100644 --- a/components/dataflow/tft/component.yaml +++ b/components/dataflow/tft/component.yaml @@ -12,7 +12,7 @@ outputs: - {name: Transformed data dir, type: GCSPath} # type: {GCSPath: {path_type: Directory}} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, /ml/transform.py] args: [ --train, {inputValue: Training data file pattern}, diff --git a/components/gcp/bigquery/query/README.md b/components/gcp/bigquery/query/README.md index ce206cfb73a4..abdd2e71645d 100644 --- a/components/gcp/bigquery/query/README.md +++ b/components/gcp/bigquery/query/README.md @@ -89,7 +89,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp bigquery_query_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/bigquery/query/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/bigquery/query/component.yaml') help(bigquery_query_op) ``` diff --git a/components/gcp/bigquery/query/component.yaml b/components/gcp/bigquery/query/component.yaml index 52b870835c56..8c93e658ac07 100644 --- a/components/gcp/bigquery/query/component.yaml +++ b/components/gcp/bigquery/query/component.yaml @@ -57,7 +57,7 @@ outputs: type: GCSPath implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.bigquery, query, --query, {inputValue: query}, diff --git a/components/gcp/bigquery/query/sample.ipynb b/components/gcp/bigquery/query/sample.ipynb index 8404214c52d2..6e54c5e71af3 100644 --- a/components/gcp/bigquery/query/sample.ipynb +++ b/components/gcp/bigquery/query/sample.ipynb @@ -108,7 +108,7 @@ "import kfp.components as comp\n", "\n", "bigquery_query_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/bigquery/query/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/bigquery/query/component.yaml')\n", "help(bigquery_query_op)" ] }, diff --git a/components/gcp/dataflow/launch_python/README.md b/components/gcp/dataflow/launch_python/README.md index 9926315e3607..9c4d73712840 100644 --- a/components/gcp/dataflow/launch_python/README.md +++ b/components/gcp/dataflow/launch_python/README.md @@ -77,7 +77,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataflow_python_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataflow/launch_python/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataflow/launch_python/component.yaml') help(dataflow_python_op) ``` diff --git a/components/gcp/dataflow/launch_python/component.yaml b/components/gcp/dataflow/launch_python/component.yaml index d44fb8fd95d4..f3c6baa3203c 100644 --- a/components/gcp/dataflow/launch_python/component.yaml +++ b/components/gcp/dataflow/launch_python/component.yaml @@ -51,7 +51,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataflow, launch_python, --python_file_path, {inputValue: python_file_path}, diff --git a/components/gcp/dataflow/launch_python/sample.ipynb b/components/gcp/dataflow/launch_python/sample.ipynb index 625b3442d1c5..9f837167fb8c 100644 --- a/components/gcp/dataflow/launch_python/sample.ipynb +++ b/components/gcp/dataflow/launch_python/sample.ipynb @@ -95,7 +95,7 @@ "import kfp.components as comp\n", "\n", "dataflow_python_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataflow/launch_python/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataflow/launch_python/component.yaml')\n", "help(dataflow_python_op)" ] }, diff --git a/components/gcp/dataflow/launch_template/README.md b/components/gcp/dataflow/launch_template/README.md index 1e16a9c357e8..67ea597d6ae8 100644 --- a/components/gcp/dataflow/launch_template/README.md +++ b/components/gcp/dataflow/launch_template/README.md @@ -67,7 +67,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataflow_template_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataflow/launch_template/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataflow/launch_template/component.yaml') help(dataflow_template_op) ``` diff --git a/components/gcp/dataflow/launch_template/component.yaml b/components/gcp/dataflow/launch_template/component.yaml index 5cc965f55cc7..5eb56f3732b6 100644 --- a/components/gcp/dataflow/launch_template/component.yaml +++ b/components/gcp/dataflow/launch_template/component.yaml @@ -61,7 +61,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataflow, launch_template, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataflow/launch_template/sample.ipynb b/components/gcp/dataflow/launch_template/sample.ipynb index b50add844df5..f576a7f4c5c1 100644 --- a/components/gcp/dataflow/launch_template/sample.ipynb +++ b/components/gcp/dataflow/launch_template/sample.ipynb @@ -85,7 +85,7 @@ "import kfp.components as comp\n", "\n", "dataflow_template_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataflow/launch_template/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataflow/launch_template/component.yaml')\n", "help(dataflow_template_op)" ] }, diff --git a/components/gcp/dataproc/create_cluster/README.md b/components/gcp/dataproc/create_cluster/README.md index cb949f25d5a5..7a2323bc086f 100644 --- a/components/gcp/dataproc/create_cluster/README.md +++ b/components/gcp/dataproc/create_cluster/README.md @@ -74,7 +74,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_create_cluster_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/create_cluster/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/create_cluster/component.yaml') help(dataproc_create_cluster_op) ``` diff --git a/components/gcp/dataproc/create_cluster/component.yaml b/components/gcp/dataproc/create_cluster/component.yaml index 5acb08dbb386..6a567356375c 100644 --- a/components/gcp/dataproc/create_cluster/component.yaml +++ b/components/gcp/dataproc/create_cluster/component.yaml @@ -68,7 +68,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, create_cluster, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/create_cluster/sample.ipynb b/components/gcp/dataproc/create_cluster/sample.ipynb index 7b5a71a9080b..6e2f6ff47138 100644 --- a/components/gcp/dataproc/create_cluster/sample.ipynb +++ b/components/gcp/dataproc/create_cluster/sample.ipynb @@ -92,7 +92,7 @@ "import kfp.components as comp\n", "\n", "dataproc_create_cluster_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/create_cluster/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/create_cluster/component.yaml')\n", "help(dataproc_create_cluster_op)" ] }, diff --git a/components/gcp/dataproc/delete_cluster/README.md b/components/gcp/dataproc/delete_cluster/README.md index 4a0ab7b24d4a..8dfd6344a55b 100644 --- a/components/gcp/dataproc/delete_cluster/README.md +++ b/components/gcp/dataproc/delete_cluster/README.md @@ -54,7 +54,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_delete_cluster_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/delete_cluster/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/delete_cluster/component.yaml') help(dataproc_delete_cluster_op) ``` diff --git a/components/gcp/dataproc/delete_cluster/component.yaml b/components/gcp/dataproc/delete_cluster/component.yaml index 6f0ec54f969f..f164bb56bbf0 100644 --- a/components/gcp/dataproc/delete_cluster/component.yaml +++ b/components/gcp/dataproc/delete_cluster/component.yaml @@ -36,7 +36,7 @@ inputs: type: Integer implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, delete_cluster, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/delete_cluster/sample.ipynb b/components/gcp/dataproc/delete_cluster/sample.ipynb index e9030238dfeb..1f1c435188ee 100644 --- a/components/gcp/dataproc/delete_cluster/sample.ipynb +++ b/components/gcp/dataproc/delete_cluster/sample.ipynb @@ -73,7 +73,7 @@ "import kfp.components as comp\n", "\n", "dataproc_delete_cluster_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/delete_cluster/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/delete_cluster/component.yaml')\n", "help(dataproc_delete_cluster_op)" ] }, diff --git a/components/gcp/dataproc/submit_hadoop_job/README.md b/components/gcp/dataproc/submit_hadoop_job/README.md index f514fe2af129..3092be008e8b 100644 --- a/components/gcp/dataproc/submit_hadoop_job/README.md +++ b/components/gcp/dataproc/submit_hadoop_job/README.md @@ -72,7 +72,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_submit_hadoop_job_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_hadoop_job/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_hadoop_job/component.yaml') help(dataproc_submit_hadoop_job_op) ``` diff --git a/components/gcp/dataproc/submit_hadoop_job/component.yaml b/components/gcp/dataproc/submit_hadoop_job/component.yaml index 716b9123bd08..617a9053862c 100644 --- a/components/gcp/dataproc/submit_hadoop_job/component.yaml +++ b/components/gcp/dataproc/submit_hadoop_job/component.yaml @@ -78,7 +78,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, submit_hadoop_job, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/submit_hadoop_job/sample.ipynb b/components/gcp/dataproc/submit_hadoop_job/sample.ipynb index 5f99eea048e9..112a921958c8 100644 --- a/components/gcp/dataproc/submit_hadoop_job/sample.ipynb +++ b/components/gcp/dataproc/submit_hadoop_job/sample.ipynb @@ -90,7 +90,7 @@ "import kfp.components as comp\n", "\n", "dataproc_submit_hadoop_job_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_hadoop_job/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_hadoop_job/component.yaml')\n", "help(dataproc_submit_hadoop_job_op)" ] }, diff --git a/components/gcp/dataproc/submit_hive_job/README.md b/components/gcp/dataproc/submit_hive_job/README.md index 3c40e59aaec6..db861d0fac8b 100644 --- a/components/gcp/dataproc/submit_hive_job/README.md +++ b/components/gcp/dataproc/submit_hive_job/README.md @@ -63,7 +63,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_submit_hive_job_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_hive_job/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_hive_job/component.yaml') help(dataproc_submit_hive_job_op) ``` diff --git a/components/gcp/dataproc/submit_hive_job/component.yaml b/components/gcp/dataproc/submit_hive_job/component.yaml index e8b655a3027e..9f0063f927af 100644 --- a/components/gcp/dataproc/submit_hive_job/component.yaml +++ b/components/gcp/dataproc/submit_hive_job/component.yaml @@ -73,7 +73,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, submit_hive_job, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/submit_hive_job/sample.ipynb b/components/gcp/dataproc/submit_hive_job/sample.ipynb index 7170b9cd0af6..b0563bec48f4 100644 --- a/components/gcp/dataproc/submit_hive_job/sample.ipynb +++ b/components/gcp/dataproc/submit_hive_job/sample.ipynb @@ -81,7 +81,7 @@ "import kfp.components as comp\n", "\n", "dataproc_submit_hive_job_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_hive_job/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_hive_job/component.yaml')\n", "help(dataproc_submit_hive_job_op)" ] }, diff --git a/components/gcp/dataproc/submit_pig_job/README.md b/components/gcp/dataproc/submit_pig_job/README.md index 4d0c4cb84805..087d9935be31 100644 --- a/components/gcp/dataproc/submit_pig_job/README.md +++ b/components/gcp/dataproc/submit_pig_job/README.md @@ -66,7 +66,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_submit_pig_job_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_pig_job/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_pig_job/component.yaml') help(dataproc_submit_pig_job_op) ``` diff --git a/components/gcp/dataproc/submit_pig_job/component.yaml b/components/gcp/dataproc/submit_pig_job/component.yaml index 77dd401727a5..80c80dc165b3 100644 --- a/components/gcp/dataproc/submit_pig_job/component.yaml +++ b/components/gcp/dataproc/submit_pig_job/component.yaml @@ -73,7 +73,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, submit_pig_job, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/submit_pig_job/sample.ipynb b/components/gcp/dataproc/submit_pig_job/sample.ipynb index 5b5c1431bd9c..5ae5118be401 100644 --- a/components/gcp/dataproc/submit_pig_job/sample.ipynb +++ b/components/gcp/dataproc/submit_pig_job/sample.ipynb @@ -84,7 +84,7 @@ "import kfp.components as comp\n", "\n", "dataproc_submit_pig_job_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_pig_job/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_pig_job/component.yaml')\n", "help(dataproc_submit_pig_job_op)" ] }, diff --git a/components/gcp/dataproc/submit_pyspark_job/README.md b/components/gcp/dataproc/submit_pyspark_job/README.md index 5b470bcb38f4..4db980d4e1c0 100644 --- a/components/gcp/dataproc/submit_pyspark_job/README.md +++ b/components/gcp/dataproc/submit_pyspark_job/README.md @@ -67,7 +67,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_submit_pyspark_job_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_pyspark_job/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_pyspark_job/component.yaml') help(dataproc_submit_pyspark_job_op) ``` diff --git a/components/gcp/dataproc/submit_pyspark_job/component.yaml b/components/gcp/dataproc/submit_pyspark_job/component.yaml index e34d0dda6901..000958a65955 100644 --- a/components/gcp/dataproc/submit_pyspark_job/component.yaml +++ b/components/gcp/dataproc/submit_pyspark_job/component.yaml @@ -67,7 +67,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, submit_pyspark_job, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/submit_pyspark_job/sample.ipynb b/components/gcp/dataproc/submit_pyspark_job/sample.ipynb index 799735014142..be999d5eec1e 100644 --- a/components/gcp/dataproc/submit_pyspark_job/sample.ipynb +++ b/components/gcp/dataproc/submit_pyspark_job/sample.ipynb @@ -86,7 +86,7 @@ "import kfp.components as comp\n", "\n", "dataproc_submit_pyspark_job_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_pyspark_job/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_pyspark_job/component.yaml')\n", "help(dataproc_submit_pyspark_job_op)" ] }, diff --git a/components/gcp/dataproc/submit_spark_job/README.md b/components/gcp/dataproc/submit_spark_job/README.md index 981dfb790fff..9e5e6bd5740a 100644 --- a/components/gcp/dataproc/submit_spark_job/README.md +++ b/components/gcp/dataproc/submit_spark_job/README.md @@ -80,7 +80,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_submit_spark_job_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_spark_job/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_spark_job/component.yaml') help(dataproc_submit_spark_job_op) ``` diff --git a/components/gcp/dataproc/submit_spark_job/component.yaml b/components/gcp/dataproc/submit_spark_job/component.yaml index e10a8f08ea54..d3f47741b3b8 100644 --- a/components/gcp/dataproc/submit_spark_job/component.yaml +++ b/components/gcp/dataproc/submit_spark_job/component.yaml @@ -74,7 +74,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, submit_spark_job, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/submit_spark_job/sample.ipynb b/components/gcp/dataproc/submit_spark_job/sample.ipynb index 8ae19bc0f7db..2200cde50b79 100644 --- a/components/gcp/dataproc/submit_spark_job/sample.ipynb +++ b/components/gcp/dataproc/submit_spark_job/sample.ipynb @@ -99,7 +99,7 @@ "import kfp.components as comp\n", "\n", "dataproc_submit_spark_job_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_spark_job/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_spark_job/component.yaml')\n", "help(dataproc_submit_spark_job_op)" ] }, diff --git a/components/gcp/dataproc/submit_sparksql_job/README.md b/components/gcp/dataproc/submit_sparksql_job/README.md index 799d93986b22..87f40b8cead6 100644 --- a/components/gcp/dataproc/submit_sparksql_job/README.md +++ b/components/gcp/dataproc/submit_sparksql_job/README.md @@ -62,7 +62,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp dataproc_submit_sparksql_job_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_sparksql_job/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_sparksql_job/component.yaml') help(dataproc_submit_sparksql_job_op) ``` diff --git a/components/gcp/dataproc/submit_sparksql_job/component.yaml b/components/gcp/dataproc/submit_sparksql_job/component.yaml index f827c73709d4..f599655b001c 100644 --- a/components/gcp/dataproc/submit_sparksql_job/component.yaml +++ b/components/gcp/dataproc/submit_sparksql_job/component.yaml @@ -73,7 +73,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.dataproc, submit_sparksql_job, --project_id, {inputValue: project_id}, diff --git a/components/gcp/dataproc/submit_sparksql_job/sample.ipynb b/components/gcp/dataproc/submit_sparksql_job/sample.ipynb index e6d561fda331..39c3aa7afecc 100644 --- a/components/gcp/dataproc/submit_sparksql_job/sample.ipynb +++ b/components/gcp/dataproc/submit_sparksql_job/sample.ipynb @@ -81,7 +81,7 @@ "import kfp.components as comp\n", "\n", "dataproc_submit_sparksql_job_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataproc/submit_sparksql_job/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataproc/submit_sparksql_job/component.yaml')\n", "help(dataproc_submit_sparksql_job_op)" ] }, diff --git a/components/gcp/ml_engine/batch_predict/README.md b/components/gcp/ml_engine/batch_predict/README.md index dc220c1aefbf..fab5b20b9fb7 100644 --- a/components/gcp/ml_engine/batch_predict/README.md +++ b/components/gcp/ml_engine/batch_predict/README.md @@ -94,7 +94,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp mlengine_batch_predict_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/batch_predict/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/batch_predict/component.yaml') help(mlengine_batch_predict_op) ``` diff --git a/components/gcp/ml_engine/batch_predict/component.yaml b/components/gcp/ml_engine/batch_predict/component.yaml index 2a36247f85f7..a4a723b1ff44 100644 --- a/components/gcp/ml_engine/batch_predict/component.yaml +++ b/components/gcp/ml_engine/batch_predict/component.yaml @@ -67,7 +67,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.ml_engine, batch_predict, --project_id, {inputValue: project_id}, diff --git a/components/gcp/ml_engine/batch_predict/sample.ipynb b/components/gcp/ml_engine/batch_predict/sample.ipynb index 2f0403b50456..61d0a5c72d33 100644 --- a/components/gcp/ml_engine/batch_predict/sample.ipynb +++ b/components/gcp/ml_engine/batch_predict/sample.ipynb @@ -112,7 +112,7 @@ "import kfp.components as comp\n", "\n", "mlengine_batch_predict_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/batch_predict/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/batch_predict/component.yaml')\n", "help(mlengine_batch_predict_op)" ] }, diff --git a/components/gcp/ml_engine/deploy/README.md b/components/gcp/ml_engine/deploy/README.md index e83adf813c55..11299d6d933f 100644 --- a/components/gcp/ml_engine/deploy/README.md +++ b/components/gcp/ml_engine/deploy/README.md @@ -110,7 +110,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp mlengine_deploy_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/deploy/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/deploy/component.yaml') help(mlengine_deploy_op) ``` diff --git a/components/gcp/ml_engine/deploy/component.yaml b/components/gcp/ml_engine/deploy/component.yaml index 2208c994a311..400145bd8955 100644 --- a/components/gcp/ml_engine/deploy/component.yaml +++ b/components/gcp/ml_engine/deploy/component.yaml @@ -93,7 +93,7 @@ outputs: type: String implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.ml_engine, deploy, --model_uri, {inputValue: model_uri}, diff --git a/components/gcp/ml_engine/deploy/sample.ipynb b/components/gcp/ml_engine/deploy/sample.ipynb index 2235cb46dfc5..dd0f9a213e55 100644 --- a/components/gcp/ml_engine/deploy/sample.ipynb +++ b/components/gcp/ml_engine/deploy/sample.ipynb @@ -128,7 +128,7 @@ "import kfp.components as comp\n", "\n", "mlengine_deploy_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/deploy/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/deploy/component.yaml')\n", "help(mlengine_deploy_op)" ] }, diff --git a/components/gcp/ml_engine/train/README.md b/components/gcp/ml_engine/train/README.md index 059eca409739..fe65e893e2c8 100644 --- a/components/gcp/ml_engine/train/README.md +++ b/components/gcp/ml_engine/train/README.md @@ -86,7 +86,7 @@ KFP_PACKAGE = 'https://storage.googleapis.com/ml-pipeline/release/0.1.14/kfp.tar import kfp.components as comp mlengine_train_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/train/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/train/component.yaml') help(mlengine_train_op) ``` diff --git a/components/gcp/ml_engine/train/component.yaml b/components/gcp/ml_engine/train/component.yaml index 39c925e49ddb..8dddf112464c 100644 --- a/components/gcp/ml_engine/train/component.yaml +++ b/components/gcp/ml_engine/train/component.yaml @@ -101,7 +101,7 @@ outputs: type: GCSPath implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-gcp:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-gcp:fe639f41661d8e17fcda64ff8242127620b80ba0 args: [ kfp_component.google.ml_engine, train, --project_id, {inputValue: project_id}, diff --git a/components/gcp/ml_engine/train/sample.ipynb b/components/gcp/ml_engine/train/sample.ipynb index 31ed940595b1..251035d4889a 100644 --- a/components/gcp/ml_engine/train/sample.ipynb +++ b/components/gcp/ml_engine/train/sample.ipynb @@ -104,7 +104,7 @@ "import kfp.components as comp\n", "\n", "mlengine_train_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/train/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/train/component.yaml')\n", "help(mlengine_train_op)" ] }, diff --git a/components/kubeflow/deployer/component.yaml b/components/kubeflow/deployer/component.yaml index c8514685f3a9..4cc031608f1f 100644 --- a/components/kubeflow/deployer/component.yaml +++ b/components/kubeflow/deployer/component.yaml @@ -11,7 +11,7 @@ inputs: # - {name: Endppoint URI, type: Serving URI, description: 'URI of the deployed prediction service..'} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [/bin/deploy.sh] args: [ --model-export-path, {inputValue: Model dir}, diff --git a/components/kubeflow/dnntrainer/component.yaml b/components/kubeflow/dnntrainer/component.yaml index 175b16e1dc68..8bbd0007538d 100644 --- a/components/kubeflow/dnntrainer/component.yaml +++ b/components/kubeflow/dnntrainer/component.yaml @@ -15,7 +15,7 @@ outputs: - {name: Training output dir, type: GCSPath, description: 'GCS or local directory.'} # type: {GCSPath: {path_type: Directory}} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, -m, trainer.task] args: [ --transformed-data-dir, {inputValue: Transformed data dir}, diff --git a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py index 2e1199f865cd..64a524c8a4d5 100644 --- a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py +++ b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py @@ -17,7 +17,7 @@ def kubeflow_tfjob_launcher_op(container_image, command, number_of_workers: int, number_of_parameter_servers: int, tfjob_timeout_minutes: int, output_dir=None, step_name='TFJob-launcher'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:ac833a084b32324b56ca56e9109e05cde02816a4', + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments = [ '--workers', number_of_workers, '--pss', number_of_parameter_servers, diff --git a/components/kubeflow/launcher/src/train.template.yaml b/components/kubeflow/launcher/src/train.template.yaml index 522d2f58fa76..46ae538d0a37 100644 --- a/components/kubeflow/launcher/src/train.template.yaml +++ b/components/kubeflow/launcher/src/train.template.yaml @@ -26,7 +26,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:fe639f41661d8e17fcda64ff8242127620b80ba0 command: - python - -m @@ -49,7 +49,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:fe639f41661d8e17fcda64ff8242127620b80ba0 command: - python - -m @@ -72,7 +72,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:fe639f41661d8e17fcda64ff8242127620b80ba0 command: - python - -m diff --git a/components/local/confusion_matrix/component.yaml b/components/local/confusion_matrix/component.yaml index 4d456434c7fb..439d00299134 100644 --- a/components/local/confusion_matrix/component.yaml +++ b/components/local/confusion_matrix/component.yaml @@ -9,7 +9,7 @@ inputs: # - {name: Metrics, type: Metrics} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, /ml/confusion_matrix.py] args: [ --predictions, {inputValue: Predictions}, diff --git a/components/local/roc/component.yaml b/components/local/roc/component.yaml index 7712cf80e8e5..452dbe2a40aa 100644 --- a/components/local/roc/component.yaml +++ b/components/local/roc/component.yaml @@ -11,7 +11,7 @@ inputs: # - {name: Metrics, type: Metrics} implementation: container: - image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:ac833a084b32324b56ca56e9109e05cde02816a4 + image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:fe639f41661d8e17fcda64ff8242127620b80ba0 command: [python2, /ml/roc.py] args: [ --predictions, {inputValue: Predictions dir}, diff --git a/samples/ai-platform/Chicago Crime Pipeline.ipynb b/samples/ai-platform/Chicago Crime Pipeline.ipynb index a6b0ee31592d..17dc112b1111 100644 --- a/samples/ai-platform/Chicago Crime Pipeline.ipynb +++ b/samples/ai-platform/Chicago Crime Pipeline.ipynb @@ -112,7 +112,7 @@ "outputs": [], "source": [ "bigquery_query_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/bigquery/query/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/bigquery/query/component.yaml')\n", "\n", "QUERY = \"\"\"\n", " SELECT count(*) as count, TIMESTAMP_TRUNC(date, DAY) as day\n", @@ -148,7 +148,7 @@ "outputs": [], "source": [ "mlengine_train_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/train/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/train/component.yaml')\n", "\n", "def train(project_id,\n", " trainer_args,\n", @@ -186,7 +186,7 @@ "outputs": [], "source": [ "mlengine_deploy_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/deploy/component.yaml')\n", + " 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/deploy/component.yaml')\n", "\n", "def deploy(\n", " project_id,\n", diff --git a/samples/kubeflow-tf/kubeflow-training-classification.py b/samples/kubeflow-tf/kubeflow-training-classification.py index fde2146054d5..a8d43f899975 100755 --- a/samples/kubeflow-tf/kubeflow-training-classification.py +++ b/samples/kubeflow-tf/kubeflow-training-classification.py @@ -19,10 +19,10 @@ from kfp import dsl from kfp import gcp -dataflow_tf_transform_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/dataflow/tft/component.yaml') -kubeflow_tf_training_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/kubeflow/dnntrainer/component.yaml') -dataflow_tf_predict_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/dataflow/predict/component.yaml') -confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/local/confusion_matrix/component.yaml') +dataflow_tf_transform_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/dataflow/tft/component.yaml') +kubeflow_tf_training_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/kubeflow/dnntrainer/component.yaml') +dataflow_tf_predict_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/dataflow/predict/component.yaml') +confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/local/confusion_matrix/component.yaml') @dsl.pipeline( name='TF training and prediction pipeline', @@ -68,7 +68,7 @@ def kubeflow_training(output, project, ).apply(gcp.use_gcp_secret('user-gcp-sa')) if use_gpu: - training.image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:ac833a084b32324b56ca56e9109e05cde02816a4', + training.image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:fe639f41661d8e17fcda64ff8242127620b80ba0', training.set_gpu_limit(1) prediction = dataflow_tf_predict_op( diff --git a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb index df829d162689..a8244a68ee79 100644 --- a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb +++ b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb @@ -43,13 +43,13 @@ "EVAL_DATA = 'gs://ml-pipeline-playground/tfx/taxi-cab-classification/eval.csv'\n", "HIDDEN_LAYER_SIZE = '1500'\n", "STEPS = 3000\n", - "DATAFLOW_TFDV_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:ac833a084b32324b56ca56e9109e05cde02816a4'\n", - "DATAFLOW_TFT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:ac833a084b32324b56ca56e9109e05cde02816a4'\n", - "DATAFLOW_TFMA_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:ac833a084b32324b56ca56e9109e05cde02816a4'\n", - "DATAFLOW_TF_PREDICT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:ac833a084b32324b56ca56e9109e05cde02816a4'\n", - "KUBEFLOW_TF_TRAINER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:ac833a084b32324b56ca56e9109e05cde02816a4'\n", - "KUBEFLOW_TF_TRAINER_GPU_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:ac833a084b32324b56ca56e9109e05cde02816a4'\n", - "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:ac833a084b32324b56ca56e9109e05cde02816a4'\n", + "DATAFLOW_TFDV_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", + "DATAFLOW_TFT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", + "DATAFLOW_TFMA_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", + "DATAFLOW_TF_PREDICT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", + "KUBEFLOW_TF_TRAINER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", + "KUBEFLOW_TF_TRAINER_GPU_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", + "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:fe639f41661d8e17fcda64ff8242127620b80ba0'\n", "DEPLOYER_MODEL = 'notebook_tfx_taxi'\n", "DEPLOYER_VERSION_DEV = 'dev'\n", "DEPLOYER_VERSION_PROD = 'prod'\n", diff --git a/samples/resnet-cmle/resnet-train-pipeline.py b/samples/resnet-cmle/resnet-train-pipeline.py index 969b3296d5ac..6ef4e012559f 100644 --- a/samples/resnet-cmle/resnet-train-pipeline.py +++ b/samples/resnet-cmle/resnet-train-pipeline.py @@ -22,11 +22,11 @@ import os dataflow_python_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/dataflow/launch_python/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/dataflow/launch_python/component.yaml') cloudml_train_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/train/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/train/component.yaml') cloudml_deploy_op = comp.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/gcp/ml_engine/deploy/component.yaml') + 'https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/gcp/ml_engine/deploy/component.yaml') def resnet_preprocess_op(project_id: 'GcpProject', output: 'GcsUri', staging_dir: 'GcsUri', train_csv: 'GcsUri[text/csv]', diff --git a/samples/tfx/taxi-cab-classification-pipeline.py b/samples/tfx/taxi-cab-classification-pipeline.py index e53646969c49..69258bcc0a45 100755 --- a/samples/tfx/taxi-cab-classification-pipeline.py +++ b/samples/tfx/taxi-cab-classification-pipeline.py @@ -22,16 +22,16 @@ platform = 'GCP' -dataflow_tf_data_validation_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/dataflow/tfdv/component.yaml') -dataflow_tf_transform_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/dataflow/tft/component.yaml') -tf_train_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/kubeflow/dnntrainer/component.yaml') -dataflow_tf_model_analyze_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/dataflow/tfma/component.yaml') -dataflow_tf_predict_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/dataflow/predict/component.yaml') +dataflow_tf_data_validation_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/dataflow/tfdv/component.yaml') +dataflow_tf_transform_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/dataflow/tft/component.yaml') +tf_train_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/kubeflow/dnntrainer/component.yaml') +dataflow_tf_model_analyze_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/dataflow/tfma/component.yaml') +dataflow_tf_predict_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/dataflow/predict/component.yaml') -confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/local/confusion_matrix/component.yaml') -roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/local/roc/component.yaml') +confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/local/confusion_matrix/component.yaml') +roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/local/roc/component.yaml') -kubeflow_deploy_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/kubeflow/deployer/component.yaml') +kubeflow_deploy_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/kubeflow/deployer/component.yaml') @dsl.pipeline( name='TFX Taxi Cab Classification Pipeline Example', diff --git a/samples/xgboost-spark/xgboost-training-cm.py b/samples/xgboost-spark/xgboost-training-cm.py index 478481c765b6..5a1f28936b81 100755 --- a/samples/xgboost-spark/xgboost-training-cm.py +++ b/samples/xgboost-spark/xgboost-training-cm.py @@ -20,8 +20,8 @@ from kfp import dsl from kfp import gcp -confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/local/confusion_matrix/component.yaml') -roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1f65a564d4d44fa5a0dc6c59929ca2211ebb3d1c/components/local/roc/component.yaml') +confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/local/confusion_matrix/component.yaml') +roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/0b07e456b1f319d8b7a7301274f55c00fda9f537/components/local/roc/component.yaml') # ! Please do not forget to enable the Dataproc API in your cluster https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview @@ -36,7 +36,7 @@ def dataproc_create_cluster_op( ): return dsl.ContainerOp( name='Dataproc - Create cluster', - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:ac833a084b32324b56ca56e9109e05cde02816a4', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments=[ '--project', project, '--region', region, @@ -56,7 +56,7 @@ def dataproc_delete_cluster_op( ): return dsl.ContainerOp( name='Dataproc - Delete cluster', - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:ac833a084b32324b56ca56e9109e05cde02816a4', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments=[ '--project', project, '--region', region, @@ -76,7 +76,7 @@ def dataproc_analyze_op( ): return dsl.ContainerOp( name='Dataproc - Analyze', - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:ac833a084b32324b56ca56e9109e05cde02816a4', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments=[ '--project', project, '--region', region, @@ -103,7 +103,7 @@ def dataproc_transform_op( ): return dsl.ContainerOp( name='Dataproc - Transform', - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:ac833a084b32324b56ca56e9109e05cde02816a4', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments=[ '--project', project, '--region', region, @@ -141,7 +141,7 @@ def dataproc_train_op( return dsl.ContainerOp( name='Dataproc - Train XGBoost model', - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-train:ac833a084b32324b56ca56e9109e05cde02816a4', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-train:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments=[ '--project', project, '--region', region, @@ -174,7 +174,7 @@ def dataproc_predict_op( ): return dsl.ContainerOp( name='Dataproc - Predict with XGBoost model', - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:ac833a084b32324b56ca56e9109e05cde02816a4', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:fe639f41661d8e17fcda64ff8242127620b80ba0', arguments=[ '--project', project, '--region', region, From a4813fffb8778efb6b445e402180391c55c82e41 Mon Sep 17 00:00:00 2001 From: Kirin Patel Date: Thu, 25 Jul 2019 18:10:05 +0000 Subject: [PATCH 07/14] Added visualization API service to frontend (#1675) --- frontend/src/lib/Apis.test.ts | 4 ++++ frontend/src/lib/Apis.ts | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/frontend/src/lib/Apis.test.ts b/frontend/src/lib/Apis.test.ts index b20c48e5c548..d43e9b256952 100644 --- a/frontend/src/lib/Apis.test.ts +++ b/frontend/src/lib/Apis.test.ts @@ -40,6 +40,10 @@ describe('Apis', () => { it('hosts a singleton runServiceApi', () => { expect(Apis.runServiceApi).toBe(Apis.runServiceApi); }); + + it('hosts a singleton visualizationServiceApi', () => { + expect(Apis.visualizationServiceApi).toBe(Apis.visualizationServiceApi); + }); it('getPodLogs', async () => { const spy = fetchSpy('http://some/address'); diff --git a/frontend/src/lib/Apis.ts b/frontend/src/lib/Apis.ts index e300ce462586..7e43908ca744 100644 --- a/frontend/src/lib/Apis.ts +++ b/frontend/src/lib/Apis.ts @@ -18,6 +18,7 @@ import { JobServiceApi } from '../apis/job'; import { RunServiceApi } from '../apis/run'; import { PipelineServiceApi, ApiPipeline } from '../apis/pipeline'; import { StoragePath } from './WorkflowParser'; +import { VisualizationServiceApi } from '../apis/visualization'; const v1beta1Prefix = 'apis/v1beta1'; @@ -79,6 +80,13 @@ export class Apis { return this._runServiceApi; } + public static get visualizationServiceApi(): VisualizationServiceApi { + if (!this._visualizationServiceApi) { + this._visualizationServiceApi = new VisualizationServiceApi({ basePath: this.basePath }); + } + return this._visualizationServiceApi; + } + /** * Retrieve various information about the build. */ @@ -158,6 +166,7 @@ export class Apis { private static _jobServiceApi?: JobServiceApi; private static _pipelineServiceApi?: PipelineServiceApi; private static _runServiceApi?: RunServiceApi; + private static _visualizationServiceApi?: VisualizationServiceApi; /** * This function will call this._fetch() and parse the resulting JSON into an object of type T. From bb339a95e4885b5ad7d99c89d31a86585b37e977 Mon Sep 17 00:00:00 2001 From: Jiaxiao Zheng Date: Thu, 25 Jul 2019 16:17:57 -0700 Subject: [PATCH 08/14] Remove redundant import. (#1656) --- .../KubeFlow Pipeline Using TFX OSS Components.ipynb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb index a8244a68ee79..ee09de2685d4 100644 --- a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb +++ b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb @@ -140,9 +140,6 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp.dsl as dsl\n", - "\n", - "\n", "# Below are a list of helper functions to wrap the components to provide a simpler interface for pipeline function.\n", "def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'GcsUri', column_names: 'GcsUri[text/json]', key_columns, project: 'GcpProject', mode, validation_output: 'GcsUri[Directory]', step_name='validation'):\n", " return dsl.ContainerOp(\n", @@ -434,8 +431,6 @@ "metadata": {}, "outputs": [], "source": [ - "from kfp import compiler\n", - "\n", "# The return value \"DeployerOp\" represents a step that can be used directly in a pipeline function\n", "DeployerOp = compiler.build_python_component(\n", " component_func=deploy_model,\n", @@ -476,8 +471,6 @@ "metadata": {}, "outputs": [], "source": [ - "from kfp import compiler\n", - "\n", "# The return value \"DeployerOp\" represents a step that can be used directly in a pipeline function\n", "DeployerOp = compiler.build_python_component(\n", " component_func=deploy_model,\n", @@ -590,8 +583,6 @@ "metadata": {}, "outputs": [], "source": [ - "from kfp import compiler\n", - "\n", "# The return value \"DeployerOp\" represents a step that can be used directly in a pipeline function\n", "#TODO: demonstrate the python2 support in another sample.\n", "DeployerOp = compiler.build_python_component(\n", From f6cf9c5f5572fc4beae9ef83f3a398678e2e5e47 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 Jul 2019 18:49:59 -0700 Subject: [PATCH 09/14] SDK - Lightweight - Added support for "None" default values (#1626) * SDK - Lightweight - Added support for "None" default values Previously it was impossible to pass None to components since it was being converted to the string "None". * is_required = not input.optional for now As asked by @gaoning777 --- sdk/python/kfp/components/_python_op.py | 23 +++++++++++++++---- sdk/python/kfp/components/_structures.py | 1 + sdk/python/tests/components/test_python_op.py | 9 ++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/sdk/python/kfp/components/_python_op.py b/sdk/python/kfp/components/_python_op.py index 4c3c4f2c8ed4..6e395a3fb0e6 100644 --- a/sdk/python/kfp/components/_python_op.py +++ b/sdk/python/kfp/components/_python_op.py @@ -156,8 +156,12 @@ def annotation_to_type_struct(annotation): input_spec = InputSpec( name=parameter.name, type=type_struct, - default=str(parameter.default) if parameter.default is not inspect.Parameter.empty else None, ) + if parameter.default is not inspect.Parameter.empty: + if parameter.default is None: + input_spec.optional = True + else: + input_spec.default = str(parameter.default) inputs.append(input_spec) #Analyzing the return type annotations. @@ -245,16 +249,27 @@ def _func_to_component_spec(func, extra_code='', base_image=_default_base_image, arguments = [] for input in component_spec.inputs: param_flag = "--" + input.name.replace("_", "-") + is_required = not input.optional #TODO: Make all parameters with default values optional in argparse so that the complex defaults can be preserved. line = '_parser.add_argument("{param_flag}", dest="{param_var}", type={param_type}, required={is_required}, default={default_repr})'.format( param_flag=param_flag, param_var=input.name, param_type=(input.type if input.type in ['int', 'float', 'bool'] else 'str'), - is_required=str(input.default is None), # TODO: Handle actual 'None' defaults! + is_required=str(is_required), default_repr=repr(str(input.default)) if input.default is not None else None, ) arg_parse_code_lines.append(line) - arguments.append(param_flag) - arguments.append(InputValuePlaceholder(input.name)) + if is_required: + arguments.append(param_flag) + arguments.append(InputValuePlaceholder(input.name)) + else: + arguments.append( + IfPlaceholder( + IfPlaceholderStructure( + condition=IsPresentPlaceholder(input.name), + then_value=[param_flag, InputValuePlaceholder(input.name)], + ) + ) + ) if component_spec.outputs: param_flag="----output-paths" diff --git a/sdk/python/kfp/components/_structures.py b/sdk/python/kfp/components/_structures.py index d4000ae372ad..b93945399b2f 100644 --- a/sdk/python/kfp/components/_structures.py +++ b/sdk/python/kfp/components/_structures.py @@ -21,6 +21,7 @@ 'OutputPathPlaceholder', 'ConcatPlaceholder', 'IsPresentPlaceholder', + 'IfPlaceholderStructure', 'IfPlaceholder', 'ContainerSpec', diff --git a/sdk/python/tests/components/test_python_op.py b/sdk/python/tests/components/test_python_op.py index 58f4621a2702..6df8a003b911 100644 --- a/sdk/python/tests/components/test_python_op.py +++ b/sdk/python/tests/components/test_python_op.py @@ -270,6 +270,15 @@ def add_multiply_two_numbers(a: float = 3, b: float = 5) -> NamedTuple('DummyNam self.assertEqual(component_spec.inputs[0].default, '3') self.assertEqual(component_spec.inputs[1].default, '5') + def test_handling_default_value_of_none(self): + def assert_is_none(a, b, arg=None) -> int: + assert arg is None + return 1 + + func = assert_is_none + op = comp.func_to_container_op(func, output_component_file='comp.yaml') + self.helper_test_2_in_1_out_component_using_local_call(func, op) + def test_end_to_end_python_component_pipeline_compilation(self): import kfp.components as comp From 80d16406f6dae2b5ee7ec73a47d45bc0e5f431f2 Mon Sep 17 00:00:00 2001 From: jingzhang36 Date: Fri, 26 Jul 2019 11:03:58 +0800 Subject: [PATCH 10/14] Increase KFP version in (1) sdk/python/setup.py (2) component_sdk/python/setup.py (3) manifests/base/kustomization.yaml (#1674) * Increase KFP version in sdk/python/setup.py component_sdk/python/setup.py * Increase kfp version in manifests/base/kustomization.yaml --- component_sdk/python/setup.py | 2 +- manifests/base/kustomization.yaml | 12 ++++++------ sdk/python/setup.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/component_sdk/python/setup.py b/component_sdk/python/setup.py index 720b6613b7fa..ba988792bf27 100644 --- a/component_sdk/python/setup.py +++ b/component_sdk/python/setup.py @@ -15,7 +15,7 @@ from setuptools import setup PACKAGE_NAME = "kfp-component" -VERSION = '0.1.24' +VERSION = '0.1.25' setup( name=PACKAGE_NAME, diff --git a/manifests/base/kustomization.yaml b/manifests/base/kustomization.yaml index f8541114c60f..724ba6c3ca5c 100644 --- a/manifests/base/kustomization.yaml +++ b/manifests/base/kustomization.yaml @@ -17,14 +17,14 @@ images: - name: mysql newTag: "5.6" - name: gcr.io/ml-pipeline/api-server - newTag: 0.1.24 + newTag: 0.1.25 - name: gcr.io/ml-pipeline/persistenceagent - newTag: 0.1.24 + newTag: 0.1.25 - name: gcr.io/ml-pipeline/scheduledworkflow - newTag: 0.1.24 + newTag: 0.1.25 - name: gcr.io/ml-pipeline/frontend - newTag: 0.1.24 + newTag: 0.1.25 - name: gcr.io/ml-pipeline/viewer-crd-controller - newTag: 0.1.24 + newTag: 0.1.25 - name: gcr.io/ml-pipeline/inverse-proxy-agent - newTag: 0.1.24 + newTag: 0.1.25 diff --git a/sdk/python/setup.py b/sdk/python/setup.py index dac70636a4e9..aa12d54f1d8e 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -15,7 +15,7 @@ from setuptools import setup NAME = 'kfp' -VERSION = '0.1.24' +VERSION = '0.1.25' REQUIRES = [ 'urllib3>=1.15,<1.25', #Fixing the version conflict with the "requests" package From a16f3806374034b8f9898fe33c60eba46c5ae282 Mon Sep 17 00:00:00 2001 From: jingzhang36 Date: Sat, 27 Jul 2019 01:13:59 +0800 Subject: [PATCH 11/14] Increase version in namespaced-install.yaml too (#1684) --- manifests/namespaced-install.yaml | 104 +++++++++++++++--------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/manifests/namespaced-install.yaml b/manifests/namespaced-install.yaml index e50b87ffd233..220d9ad0f53c 100644 --- a/manifests/namespaced-install.yaml +++ b/manifests/namespaced-install.yaml @@ -238,6 +238,38 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role +metadata: + name: ml-pipeline-viewer-controller-role + namespace: kubeflow +rules: +- apiGroups: + - '*' + resources: + - deployments + - services + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - kubeflow.org + resources: + - viewers + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role metadata: labels: app: ml-pipeline @@ -359,38 +391,6 @@ rules: verbs: - '*' --- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: ml-pipeline-viewer-controller-role - namespace: kubeflow -rules: -- apiGroups: - - '*' - resources: - - deployments - - services - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - kubeflow.org - resources: - - viewers - verbs: - - create - - get - - list - - watch - - update - - patch - - delete ---- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -451,6 +451,20 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding +metadata: + name: ml-pipeline-viewer-crd-binding + namespace: kubeflow +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ml-pipeline-viewer-controller-role +subjects: +- kind: ServiceAccount + name: ml-pipeline-viewer-crd-service-account + namespace: kubeflow +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding metadata: labels: app: ml-pipeline @@ -495,20 +509,6 @@ subjects: name: proxy-agent-runner namespace: kubeflow --- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: ml-pipeline-viewer-crd-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline-viewer-controller-role -subjects: -- kind: ServiceAccount - name: ml-pipeline-viewer-crd-service-account - namespace: kubeflow ---- apiVersion: v1 data: config: | @@ -662,7 +662,7 @@ spec: - env: - name: NAMESPACE value: kubeflow - image: gcr.io/ml-pipeline/persistenceagent:0.1.24 + image: gcr.io/ml-pipeline/persistenceagent:0.1.25 imagePullPolicy: IfNotPresent name: ml-pipeline-persistenceagent serviceAccountName: ml-pipeline-persistenceagent @@ -687,7 +687,7 @@ spec: - env: - name: NAMESPACE value: kubeflow - image: gcr.io/ml-pipeline/scheduledworkflow:0.1.24 + image: gcr.io/ml-pipeline/scheduledworkflow:0.1.25 imagePullPolicy: IfNotPresent name: ml-pipeline-scheduledworkflow serviceAccountName: ml-pipeline-scheduledworkflow @@ -712,7 +712,7 @@ spec: - env: - name: MINIO_NAMESPACE value: kubeflow - image: gcr.io/ml-pipeline/frontend:0.1.24 + image: gcr.io/ml-pipeline/frontend:0.1.25 imagePullPolicy: IfNotPresent name: ml-pipeline-ui ports: @@ -741,7 +741,7 @@ spec: value: kubeflow - name: MAX_NUM_VIEWERS value: "50" - image: gcr.io/ml-pipeline/viewer-crd-controller:0.1.24 + image: gcr.io/ml-pipeline/viewer-crd-controller:0.1.25 imagePullPolicy: Always name: ml-pipeline-viewer-crd serviceAccountName: ml-pipeline-viewer-crd-service-account @@ -768,7 +768,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/ml-pipeline/api-server:0.1.24 + image: gcr.io/ml-pipeline/api-server:0.1.25 imagePullPolicy: IfNotPresent name: ml-pipeline-api-server ports: @@ -826,7 +826,7 @@ spec: app: proxy-agent spec: containers: - - image: gcr.io/ml-pipeline/inverse-proxy-agent:0.1.24 + - image: gcr.io/ml-pipeline/inverse-proxy-agent:0.1.25 imagePullPolicy: IfNotPresent name: proxy-agent serviceAccountName: proxy-agent-runner From f3de0c415dcb6663687aad452f2aa7d99ca90d0f Mon Sep 17 00:00:00 2001 From: Kirin Patel Date: Fri, 26 Jul 2019 19:05:57 +0000 Subject: [PATCH 12/14] Add new PlotType to Allow for Visualization Creation (#1677) * Added VisualizationCreator as new PlotType * Addressed PR feedback made by @rileyjbauer * Fix issue where updated props would not be accessible to VisualizationCreator View [comment](https://github.com/kubeflow/pipelines/pull/1677#discussion_r307490016) for more details * Updated select and input fields to become disabled when isBusy is true * Updated test names to be more descriptive * Fixed bug where enum key was used rather than enum value when determining ApiVisualizationType --- frontend/src/components/viewers/Viewer.ts | 1 + .../components/viewers/ViewerContainer.tsx | 2 + .../viewers/VisualizationCreator.test.tsx | 215 +++++++ .../viewers/VisualizationCreator.tsx | 139 +++++ .../ViewerContainer.test.tsx.snap | 12 + .../VisualizationCreator.test.tsx.snap | 548 ++++++++++++++++++ 6 files changed, 917 insertions(+) create mode 100644 frontend/src/components/viewers/VisualizationCreator.test.tsx create mode 100644 frontend/src/components/viewers/VisualizationCreator.tsx create mode 100644 frontend/src/components/viewers/__snapshots__/VisualizationCreator.test.tsx.snap diff --git a/frontend/src/components/viewers/Viewer.ts b/frontend/src/components/viewers/Viewer.ts index 2a9470649f31..88beae2bfb69 100644 --- a/frontend/src/components/viewers/Viewer.ts +++ b/frontend/src/components/viewers/Viewer.ts @@ -22,6 +22,7 @@ export enum PlotType { ROC = 'roc', TABLE = 'table', TENSORBOARD = 'tensorboard', + VISUALIZATION_CREATOR = 'visualization-creator', WEB_APP = 'web-app', } diff --git a/frontend/src/components/viewers/ViewerContainer.tsx b/frontend/src/components/viewers/ViewerContainer.tsx index 050594a6144e..0ec673c9e3b5 100644 --- a/frontend/src/components/viewers/ViewerContainer.tsx +++ b/frontend/src/components/viewers/ViewerContainer.tsx @@ -22,6 +22,7 @@ import PagedTable from './PagedTable'; import ROCCurve from './ROCCurve'; import TensorboardViewer from './Tensorboard'; import { PlotType, ViewerConfig } from './Viewer'; +import VisualizationCreator from './VisualizationCreator'; export const componentMap = { [PlotType.CONFUSION_MATRIX]: ConfusionMatrix, @@ -29,6 +30,7 @@ export const componentMap = { [PlotType.ROC]: ROCCurve, [PlotType.TABLE]: PagedTable, [PlotType.TENSORBOARD]: TensorboardViewer, + [PlotType.VISUALIZATION_CREATOR]: VisualizationCreator, [PlotType.WEB_APP]: HTMLViewer, }; diff --git a/frontend/src/components/viewers/VisualizationCreator.test.tsx b/frontend/src/components/viewers/VisualizationCreator.test.tsx new file mode 100644 index 000000000000..29c119133f45 --- /dev/null +++ b/frontend/src/components/viewers/VisualizationCreator.test.tsx @@ -0,0 +1,215 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as React from 'react'; +import { shallow } from 'enzyme'; +import { PlotType } from './Viewer'; +import VisualizationCreator, { VisualizationCreatorConfig } from './VisualizationCreator'; +import { ApiVisualizationType } from '../../apis/visualization'; + +describe('VisualizationCreator', () => { + it('does not render component when no config is provided', () => { + const tree = shallow(); + expect(tree).toMatchSnapshot(); + }); + + it('renders component when empty config is provided', () => { + const config: VisualizationCreatorConfig = { + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + expect(tree).toMatchSnapshot(); + }); + + it('renders component when isBusy is not provided', () => { + const config: VisualizationCreatorConfig = { + onGenerate: jest.fn(), + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + expect(tree).toMatchSnapshot(); + }); + + it('renders component when onGenerate is not provided', () => { + const config: VisualizationCreatorConfig = { + isBusy: false, + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + expect(tree).toMatchSnapshot(); + }); + + it('renders component when all parameters in config are provided', () => { + const config: VisualizationCreatorConfig = { + isBusy: false, + onGenerate: jest.fn(), + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + expect(tree).toMatchSnapshot(); + }); + + it('has a disabled BusyButton if selectedType is undefined', () => { + const config: VisualizationCreatorConfig = { + isBusy: false, + onGenerate: jest.fn(), + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + inputPath: 'gs://ml-pipeline/data.csv', + }); + expect(tree.find('BusyButton').at(0).prop('disabled')).toBe(true); + }); + + it('has a disabled BusyButton if inputPath is an empty string', () => { + const config: VisualizationCreatorConfig = { + isBusy: false, + onGenerate: jest.fn(), + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + // inputPath by default is set to '' + selectedType: ApiVisualizationType.CURVE, + }); + expect(tree.find('BusyButton').at(0).prop('disabled')).toBe(true); + }); + + it('has a disabled BusyButton if onGenerate is not provided as a prop', () => { + const config: VisualizationCreatorConfig = { + isBusy: false, + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + inputPath: 'gs://ml-pipeline/data.csv', + selectedType: ApiVisualizationType.CURVE, + }); + expect(tree.find('BusyButton').at(0).prop('disabled')).toBe(true); + }); + + it('has a disabled BusyButton if isBusy is true', () => { + const config: VisualizationCreatorConfig = { + isBusy: true, + onGenerate: jest.fn(), + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + inputPath: 'gs://ml-pipeline/data.csv', + selectedType: ApiVisualizationType.CURVE, + }); + expect(tree.find('BusyButton').at(0).prop('disabled')).toBe(true); + }); + + it('has an enabled BusyButton if onGenerate is provided and inputPath and selectedType are set', () => { + const config: VisualizationCreatorConfig = { + isBusy: false, + onGenerate: jest.fn(), + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + inputPath: 'gs://ml-pipeline/data.csv', + selectedType: ApiVisualizationType.CURVE, + }); + expect(tree.find('BusyButton').at(0).prop('disabled')).toBe(false); + }); + + it('calls onGenerate when BusyButton is clicked', () => { + const onGenerate = jest.fn(); + const config: VisualizationCreatorConfig = { + isBusy: false, + onGenerate, + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + arguments: '{}', + inputPath: 'gs://ml-pipeline/data.csv', + selectedType: ApiVisualizationType.CURVE, + }); + tree.find('BusyButton').at(0).simulate('click'); + expect(onGenerate).toBeCalled(); + }); + + it('passes state as parameters to onGenerate when BusyButton is clicked', () => { + const onGenerate = jest.fn(); + const config: VisualizationCreatorConfig = { + isBusy: false, + onGenerate, + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + arguments: '{}', + inputPath: 'gs://ml-pipeline/data.csv', + selectedType: ApiVisualizationType.CURVE, + }); + tree.find('BusyButton').at(0).simulate('click'); + expect(onGenerate).toBeCalledWith('{}', 'gs://ml-pipeline/data.csv', ApiVisualizationType.CURVE); + }); + + it('renders the provided arguments correctly', () => { + const config: VisualizationCreatorConfig = { + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + arguments: JSON.stringify({is_generated: 'True'}), + }); + expect(tree).toMatchSnapshot(); + }); + + it('renders the provided input path correctly', () => { + const config: VisualizationCreatorConfig = { + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + inputPath: 'gs://ml-pipeline/data.csv', + }); + expect(tree).toMatchSnapshot(); + }); + + it('renders the selected visualization type correctly', () => { + const config: VisualizationCreatorConfig = { + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + tree.setState({ + selectedType: ApiVisualizationType.CURVE, + }); + expect(tree).toMatchSnapshot(); + }); + + it('disables all select and input fields when busy', () => { + const config: VisualizationCreatorConfig = { + isBusy: true, + type: PlotType.VISUALIZATION_CREATOR, + }; + const tree = shallow(); + // toMatchSnapshot is used rather than three individual checks for the + // disabled prop due to an issue where the Input components are not + // selectable by tree.find(). + expect(tree).toMatchSnapshot(); + }); + + it('returns friendly display name', () => { + expect(VisualizationCreator.prototype.getDisplayName()).toBe('Visualization Creator'); + }); +}); diff --git a/frontend/src/components/viewers/VisualizationCreator.tsx b/frontend/src/components/viewers/VisualizationCreator.tsx new file mode 100644 index 000000000000..17a8498584d0 --- /dev/null +++ b/frontend/src/components/viewers/VisualizationCreator.tsx @@ -0,0 +1,139 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as React from 'react'; +import BusyButton from '../../atoms/BusyButton'; +import FormControl from '@material-ui/core/FormControl'; +import Input from '../../atoms/Input'; +import InputLabel from '@material-ui/core/InputLabel'; +import MenuItem from '@material-ui/core/MenuItem'; +import Select from '@material-ui/core/Select'; +import Viewer, { ViewerConfig } from './Viewer'; +import { ApiVisualizationType } from '../../apis/visualization'; + + +export interface VisualizationCreatorConfig extends ViewerConfig { + // Whether there is currently a visualization being generated or not. + isBusy?: boolean; + // Function called to generate a visualization. + onGenerate?: (visualizationArguments: string, inputPath: string, type: ApiVisualizationType) => void; +} + +interface VisualizationCreatorProps { + configs: VisualizationCreatorConfig[]; + maxWidth?: number; +} + +interface VisualizationCreatorState { + // arguments is expected to be a JSON object in string form. + arguments: string; + inputPath: string; + selectedType?: ApiVisualizationType; +} + +class VisualizationCreator extends Viewer { + constructor(props: VisualizationCreatorProps) { + super(props); + this.state = { + arguments: '', + inputPath: '', + }; + } + + public getDisplayName(): string { + return 'Visualization Creator'; + } + + public render(): JSX.Element | null { + const { configs } = this.props; + const config = configs[0]; + const { arguments: _arguments, inputPath, selectedType } = this.state; + + if (!config) { + return null; + } + + const { isBusy = false, onGenerate } = config; + + // Only allow a visualization to be generated if one is not already being + // generated (as indicated by the isBusy tag), and if there is an inputPath + // provided, and a visualization type is selected, and a onGenerate function + // is provided. + const canGenerate = !isBusy && + !!inputPath.length && + !!selectedType && + !!onGenerate; + + return
+ + Type + + + + ) => this.setState({ inputPath: e.target.value })} /> + ) => this.setState({ arguments: e.target.value })} /> + { + if (onGenerate && selectedType) { + onGenerate(_arguments, inputPath, selectedType); + } + }} /> +
; + } +} + +export default VisualizationCreator; diff --git a/frontend/src/components/viewers/__snapshots__/ViewerContainer.test.tsx.snap b/frontend/src/components/viewers/__snapshots__/ViewerContainer.test.tsx.snap index 6d0114536b00..37d505d350ef 100644 --- a/frontend/src/components/viewers/__snapshots__/ViewerContainer.test.tsx.snap +++ b/frontend/src/components/viewers/__snapshots__/ViewerContainer.test.tsx.snap @@ -62,6 +62,18 @@ exports[`ViewerContainer renders a viewer of type TENSORBOARD 1`] = ` /> `; +exports[`ViewerContainer renders a viewer of type VISUALIZATION_CREATOR 1`] = ` + +`; + exports[`ViewerContainer renders a viewer of type WEB_APP 1`] = ` + + + Type + + + + CURVE + + + + + + + +`; + +exports[`VisualizationCreator does not render component when no config is provided 1`] = `""`; + +exports[`VisualizationCreator renders component when all parameters in config are provided 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; + +exports[`VisualizationCreator renders component when empty config is provided 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; + +exports[`VisualizationCreator renders component when isBusy is not provided 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; + +exports[`VisualizationCreator renders component when onGenerate is not provided 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; + +exports[`VisualizationCreator renders the provided arguments correctly 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; + +exports[`VisualizationCreator renders the provided input path correctly 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; + +exports[`VisualizationCreator renders the selected visualization type correctly 1`] = ` +
+ + + Type + + + + CURVE + + + + + + +
+`; From f63b57b66cf929a3551f57aaabbe6d35128d89c9 Mon Sep 17 00:00:00 2001 From: Riley Bauer <34456002+rileyjbauer@users.noreply.github.com> Date: Fri, 26 Jul 2019 14:09:58 -0700 Subject: [PATCH 13/14] Clears the workflow's name in GetWorkflowSpec and uses it for the GenerateName (#1689) --- backend/src/common/util/workflow.go | 10 +++++-- backend/src/common/util/workflow_test.go | 36 +++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/backend/src/common/util/workflow.go b/backend/src/common/util/workflow.go index c99a0be5e5cd..1ddc8a8c7581 100644 --- a/backend/src/common/util/workflow.go +++ b/backend/src/common/util/workflow.go @@ -136,7 +136,7 @@ func (w *Workflow) ScheduledAtInSecOr0() int64 { } func (w *Workflow) FinishedAt() int64 { - if w.Status.FinishedAt.IsZero(){ + if w.Status.FinishedAt.IsZero() { // If workflow is not finished return 0 } @@ -165,7 +165,13 @@ func (w *Workflow) GetWorkflowSpec() *Workflow { workflow := w.DeepCopy() workflow.Status = workflowapi.WorkflowStatus{} workflow.TypeMeta = metav1.TypeMeta{Kind: w.Kind, APIVersion: w.APIVersion} - workflow.ObjectMeta = metav1.ObjectMeta{Name: w.Name, GenerateName: w.GenerateName} + // To prevent collisions, clear name, set GenerateName to first 200 runes of previous name. + nameRunes := []rune(w.Name) + length := len(nameRunes) + if length > 200 { + length = 200 + } + workflow.ObjectMeta = metav1.ObjectMeta{GenerateName: string(nameRunes[:length])} return NewWorkflow(workflow) } diff --git a/backend/src/common/util/workflow_test.go b/backend/src/common/util/workflow_test.go index 31c9379ee1e7..c95503bea410 100644 --- a/backend/src/common/util/workflow_test.go +++ b/backend/src/common/util/workflow_test.go @@ -291,7 +291,41 @@ func TestGetWorkflowSpec(t *testing.T) { expected := &workflowapi.Workflow{ ObjectMeta: metav1.ObjectMeta{ - Name: "WORKFLOW_NAME", + GenerateName: "WORKFLOW_NAME", + }, + Spec: workflowapi.WorkflowSpec{ + Arguments: workflowapi.Arguments{ + Parameters: []workflowapi.Parameter{ + {Name: "PARAM", Value: StringPointer("VALUE")}, + }, + }, + }, + } + + assert.Equal(t, expected, workflow.GetWorkflowSpec().Get()) +} + +func TestGetWorkflowSpecTruncatesNameIfLongerThan200Runes(t *testing.T) { + workflow := NewWorkflow(&workflowapi.Workflow{ + ObjectMeta: metav1.ObjectMeta{ + Name: "THIS_NAME_IS_GREATER_THAN_200_CHARACTERS_AND_WILL_BE_TRUNCATED_AFTER_THE_X_OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOXZZZZZZZZ", + Labels: map[string]string{"key": "value"}, + }, + Spec: workflowapi.WorkflowSpec{ + Arguments: workflowapi.Arguments{ + Parameters: []workflowapi.Parameter{ + {Name: "PARAM", Value: StringPointer("VALUE")}, + }, + }, + }, + Status: workflowapi.WorkflowStatus{ + Message: "I AM A MESSAGE", + }, + }) + + expected := &workflowapi.Workflow{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "THIS_NAME_IS_GREATER_THAN_200_CHARACTERS_AND_WILL_BE_TRUNCATED_AFTER_THE_X_OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOX", }, Spec: workflowapi.WorkflowSpec{ Arguments: workflowapi.Arguments{ From 0283dddbeecadbb03f7ba0623577d748f3140b45 Mon Sep 17 00:00:00 2001 From: Ning Date: Fri, 26 Jul 2019 16:09:58 -0700 Subject: [PATCH 14/14] Separate codegen from containerbuild (#1679) * refactor component build code * remove unnecessary import * minor changes * fix unit tests * fix sample test bug * revert the change of the dependency orders * add -u to disable python stdout buffering * address the comments * separate lines to look clean * fix unit tests * fix --- sdk/python/kfp/compiler/_component_builder.py | 349 +++++++++--------- .../tests/compiler/component_builder_test.py | 289 ++++++--------- 2 files changed, 285 insertions(+), 353 deletions(-) diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/compiler/_component_builder.py index 435fd74a2deb..7bd9866d89de 100644 --- a/sdk/python/kfp/compiler/_component_builder.py +++ b/sdk/python/kfp/compiler/_component_builder.py @@ -21,7 +21,7 @@ import tempfile import logging from collections import OrderedDict -from pathlib import PurePath, Path +from pathlib import Path from ..components._components import _create_task_factory_from_component_spec class VersionedDependency(object): @@ -101,82 +101,47 @@ def generate_pip_requirements(self, target_file): version_str += ' <= ' + version.max_version + ',' f.write(name + version_str.rstrip(',') + '\n') -class DockerfileHelper(object): - """ Dockerfile Helper generates a tarball with dockerfile, ready for docker build - arc_dockerfile_name: dockerfile filename that is stored in the tarball """ - - def __init__(self, arc_dockerfile_name): - self._arc_dockerfile_name = arc_dockerfile_name - self._ARC_REQUIREMENT_FILE = 'requirements.txt' - - def _generate_pip_requirement(self, dependency, requirement_filepath): - dependency_helper = DependencyHelper() - for version in dependency: - dependency_helper.add_python_package(version) - dependency_helper.generate_pip_requirements(requirement_filepath) +def _dependency_to_requirements(dependency=[], filename='requirements.txt'): + """ + Generates a requirement file based on the dependency + Args: + dependency (list): a list of VersionedDependency, which includes the package name and versions + filename (str): requirement file name, default as requirements.txt + """ + dependency_helper = DependencyHelper() + for version in dependency: + dependency_helper.add_python_package(version) + dependency_helper.generate_pip_requirements(filename) - def _generate_dockerfile_with_py(self, target_file, base_image, python_filepath, has_requirement_file, python_version): - """ _generate_docker_file generates a simple dockerfile with the python path - args: - target_file (str): target file name for the dockerfile. +def _generate_dockerfile(filename, base_image, entrypoint_filename, python_version, requirement_filename=None): + """ + generates dockerfiles + Args: + filename (str): target file name for the dockerfile. base_image (str): the base image name. - python_filepath (str): the path of the python file that is copied to the docker image. - has_requirement_file (bool): whether it has a requirement file or not. + entrypoint_filename (str): the path of the entrypoint source file that is copied to the docker image. python_version (str): choose python2 or python3 - """ - if python_version not in ['python2', 'python3']: - raise ValueError('python_version has to be either python2 or python3') - with open(target_file, 'w') as f: - f.write('FROM ' + base_image + '\n') - if python_version is 'python3': - f.write('RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools\n') - else: - f.write('RUN apt-get update -y && apt-get install --no-install-recommends -y -q python python-pip python-setuptools\n') - if has_requirement_file: - f.write('ADD ' + self._ARC_REQUIREMENT_FILE + ' /ml/\n') - if python_version is 'python3': - f.write('RUN pip3 install -r /ml/' + self._ARC_REQUIREMENT_FILE + '\n') - else: - f.write('RUN pip install -r /ml/' + self._ARC_REQUIREMENT_FILE + '\n') - f.write('ADD ' + python_filepath + " /ml/" + '\n') + requirement_filename (str): requirement file name + """ + if python_version not in ['python2', 'python3']: + raise ValueError('python_version has to be either python2 or python3') + with open(filename, 'w') as f: + f.write('FROM ' + base_image + '\n') + if python_version is 'python3': + f.write('RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools\n') + else: + f.write('RUN apt-get update -y && apt-get install --no-install-recommends -y -q python python-pip python-setuptools\n') + if requirement_filename is not None: + f.write('ADD ' + requirement_filename + ' /ml/requirements.txt\n') if python_version is 'python3': - f.write('ENTRYPOINT ["python3", "/ml/' + python_filepath + '"]') + f.write('RUN pip3 install -r /ml/requirements.txt\n') else: - f.write('ENTRYPOINT ["python", "/ml/' + python_filepath + '"]') - - def _wrap_files_in_tarball(self, tarball_path, files={}): - """ _wrap_files_in_tarball creates a tarball for all the input files - with the filename configured as the key of files """ - if not tarball_path.endswith('.tar.gz'): - raise ValueError('the tarball path should end with .tar.gz') - with tarfile.open(tarball_path, 'w:gz') as tarball: - for key, value in files.items(): - tarball.add(value, arcname=key) - - def prepare_docker_tarball_with_py(self, arc_python_filename, python_filepath, base_image, local_tarball_path, python_version, dependency=None): - """ prepare_docker_tarball is the API to generate dockerfile and prepare the tarball with python scripts - args: - python_version (str): choose python2 or python3 - """ - if python_version not in ['python2', 'python3']: - raise ValueError('python_version has to be either python2 or python3') - with tempfile.TemporaryDirectory() as local_build_dir: - has_requirement_file = False - local_requirement_path = os.path.join(local_build_dir, self._ARC_REQUIREMENT_FILE) - if dependency is not None and len(dependency) != 0: - self._generate_pip_requirement(dependency, local_requirement_path) - has_requirement_file = True - local_dockerfile_path = os.path.join(local_build_dir, self._arc_dockerfile_name) - self._generate_dockerfile_with_py(local_dockerfile_path, base_image, arc_python_filename, has_requirement_file, python_version) - file_lists = {self._arc_dockerfile_name:local_dockerfile_path, - arc_python_filename:python_filepath} - if has_requirement_file: - file_lists[self._ARC_REQUIREMENT_FILE] = local_requirement_path - self._wrap_files_in_tarball(local_tarball_path, file_lists) - - def prepare_docker_tarball(self, dockerfile_path, local_tarball_path): - """ prepare_docker_tarball is the API to prepare a tarball with the dockerfile """ - self._wrap_files_in_tarball(local_tarball_path, {self._arc_dockerfile_name:dockerfile_path}) + f.write('RUN pip install -r /ml/requirements.txt\n') + f.write('ADD ' + entrypoint_filename + ' /ml/main.py\n') + if python_version is 'python3': + f.write('ENTRYPOINT ["python3", "-u", "/ml/main.py"]') + else: + f.write('ENTRYPOINT ["python", "-u", "/ml/main.py"]') class CodeGenerator(object): """ CodeGenerator helps to generate python codes with identation """ @@ -204,18 +169,125 @@ def end(self): line_sep = '\n' return line_sep.join(self._code) + line_sep +#TODO: currently it supports single output, future support for multiple return values +def _func_to_entrypoint(component_func, python_version='python3'): + ''' + args: + python_version (str): choose python2 or python3, default is python3 + ''' + if python_version not in ['python2', 'python3']: + raise ValueError('python_version has to be either python2 or python3') + + fullargspec = inspect.getfullargspec(component_func) + annotations = fullargspec[6] + input_args = fullargspec[0] + inputs = {} + for key, value in annotations.items(): + if key != 'return': + inputs[key] = value + if len(input_args) != len(inputs): + raise Exception('Some input arguments do not contain annotations.') + if 'return' in annotations and annotations['return'] not in [int, float, str, bool]: + raise Exception('Output type not supported and supported types are [int, float, str, bool]') + # inputs is a dictionary with key of argument name and value of type class + # output is a type class, e.g. str and int. + + # Follow the same indentation with the component source codes. + component_src = inspect.getsource(component_func) + match = re.search(r'\n([ \t]+)[\w]+', component_src) + indentation = match.group(1) if match else '\t' + codegen = CodeGenerator(indentation=indentation) + + # Function signature + new_func_name = 'wrapper_' + component_func.__name__ + codegen.begin() + func_signature = 'def ' + new_func_name + '(' + for input_arg in input_args: + func_signature += input_arg + ',' + func_signature += '_output_file):' + codegen.writeline(func_signature) + + # Call user function + codegen.indent() + call_component_func = 'output = ' + component_func.__name__ + '(' + for input_arg in input_args: + call_component_func += inputs[input_arg].__name__ + '(' + input_arg + '),' + call_component_func = call_component_func.rstrip(',') + call_component_func += ')' + codegen.writeline(call_component_func) + + # Serialize output + codegen.writeline('import os') + codegen.writeline('os.makedirs(os.path.dirname(_output_file))') + codegen.writeline('with open(_output_file, "w") as data:') + codegen.indent() + codegen.writeline('data.write(str(output))') + wrapper_code = codegen.end() + + # CLI codes + codegen.begin() + codegen.writeline('import argparse') + codegen.writeline('parser = argparse.ArgumentParser(description="Parsing arguments")') + for input_arg in input_args: + codegen.writeline('parser.add_argument("' + input_arg + '", type=' + inputs[input_arg].__name__ + ')') + codegen.writeline('parser.add_argument("_output_file", type=str)') + codegen.writeline('args = vars(parser.parse_args())') + codegen.writeline('') + codegen.writeline('if __name__ == "__main__":') + codegen.indent() + codegen.writeline(new_func_name + '(**args)') + + # Remove the decorator from the component source + src_lines = component_src.split('\n') + start_line_num = 0 + for line in src_lines: + if line.startswith('def '): + break + start_line_num += 1 + if python_version == 'python2': + src_lines[start_line_num] = 'def ' + component_func.__name__ + '(' + ', '.join((inspect.getfullargspec(component_func).args)) + '):' + dedecorated_component_src = '\n'.join(src_lines[start_line_num:]) + + complete_component_code = dedecorated_component_src + '\n' + wrapper_code + '\n' + codegen.end() + return complete_component_code + class ImageBuilder(object): """ Component Builder. """ def __init__(self, gcs_base, target_image): - self._arc_dockerfile_name = 'dockerfile' - self._arc_python_filepath = 'main.py' - self._tarball_name = str(uuid.uuid4()) + '.tar.gz' + self._arc_docker_filename = 'dockerfile' + self._arc_python_filename = 'main.py' + self._arc_requirement_filename = 'requirements.txt' + self._tarball_filename = str(uuid.uuid4()) + '.tar.gz' self._gcs_base = gcs_base if not self._check_gcs_path(self._gcs_base): raise Exception('ImageBuild __init__ failure.') - self._gcs_path = os.path.join(self._gcs_base, self._tarball_name) + self._gcs_path = os.path.join(self._gcs_base, self._tarball_filename) self._target_image = target_image + def _wrap_files_in_tarball(self, tarball_path, files={}): + """ _wrap_files_in_tarball creates a tarball for all the input files + with the filename configured as the key of files """ + if not tarball_path.endswith('.tar.gz'): + raise ValueError('the tarball path should end with .tar.gz') + with tarfile.open(tarball_path, 'w:gz') as tarball: + for key, value in files.items(): + tarball.add(value, arcname=key) + + def _prepare_buildfiles(self, local_tarball_path, docker_filename, python_filename=None, requirement_filename=None): + """ _prepare_buildfiles generates the tarball with all the build files + Args: + local_tarball_path (str): generated tarball file + docker_filename (str): docker filename + python_filename (str): python filename + requirement_filename (str): requirement filename + """ + file_lists = {self._arc_docker_filename:docker_filename} + if python_filename is not None: + file_lists[self._arc_python_filename] = python_filename + if requirement_filename is not None: + file_lists[self._arc_requirement_filename] = requirement_filename + self._wrap_files_in_tarball(local_tarball_path, file_lists) + def _check_gcs_path(self, gcs_path): """ _check_gcs_path check both the path validity and write permissions """ logging.info('Checking path: {}...'.format(gcs_path)) @@ -237,9 +309,9 @@ def _generate_kaniko_spec(self, namespace, arc_dockerfile_name, gcs_path, target 'restartPolicy': 'Never', 'containers': [{ 'name': 'kaniko', - 'args': ['--cache=true', - '--dockerfile=' + arc_dockerfile_name, - '--context=' + gcs_path, + 'args': ['--cache=true', + '--dockerfile=' + arc_dockerfile_name, + '--context=' + gcs_path, '--destination=' + target_image], 'image': 'gcr.io/kaniko-project/executor@sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400', 'env': [{ @@ -259,96 +331,13 @@ def _generate_kaniko_spec(self, namespace, arc_dockerfile_name, gcs_path, target }], 'serviceAccountName': 'default'} } - return content - #TODO: currently it supports single output, future support for multiple return values - def _generate_entrypoint(self, component_func, python_version='python3'): - ''' - args: - python_version (str): choose python2 or python3, default is python3 - ''' - if python_version not in ['python2', 'python3']: - raise ValueError('python_version has to be either python2 or python3') - - fullargspec = inspect.getfullargspec(component_func) - annotations = fullargspec[6] - input_args = fullargspec[0] - inputs = {} - for key, value in annotations.items(): - if key != 'return': - inputs[key] = value - if len(input_args) != len(inputs): - raise Exception('Some input arguments do not contain annotations.') - if 'return' in annotations and annotations['return'] not in [int, float, str, bool]: - raise Exception('Output type not supported and supported types are [int, float, str, bool]') - # inputs is a dictionary with key of argument name and value of type class - # output is a type class, e.g. str and int. - - # Follow the same indentation with the component source codes. - component_src = inspect.getsource(component_func) - match = re.search(r'\n([ \t]+)[\w]+', component_src) - indentation = match.group(1) if match else '\t' - codegen = CodeGenerator(indentation=indentation) - - # Function signature - new_func_name = 'wrapper_' + component_func.__name__ - codegen.begin() - func_signature = 'def ' + new_func_name + '(' - for input_arg in input_args: - func_signature += input_arg + ',' - func_signature += '_output_file):' - codegen.writeline(func_signature) - - # Call user function - codegen.indent() - call_component_func = 'output = ' + component_func.__name__ + '(' - for input_arg in input_args: - call_component_func += inputs[input_arg].__name__ + '(' + input_arg + '),' - call_component_func = call_component_func.rstrip(',') - call_component_func += ')' - codegen.writeline(call_component_func) - - # Serialize output - codegen.writeline('import os') - codegen.writeline('os.makedirs(os.path.dirname(_output_file))') - codegen.writeline('with open(_output_file, "w") as data:') - codegen.indent() - codegen.writeline('data.write(str(output))') - wrapper_code = codegen.end() - - # CLI codes - codegen.begin() - codegen.writeline('import argparse') - codegen.writeline('parser = argparse.ArgumentParser(description="Parsing arguments")') - for input_arg in input_args: - codegen.writeline('parser.add_argument("' + input_arg + '", type=' + inputs[input_arg].__name__ + ')') - codegen.writeline('parser.add_argument("_output_file", type=str)') - codegen.writeline('args = vars(parser.parse_args())') - codegen.writeline('') - codegen.writeline('if __name__ == "__main__":') - codegen.indent() - codegen.writeline(new_func_name + '(**args)') - - # Remove the decorator from the component source - src_lines = component_src.split('\n') - start_line_num = 0 - for line in src_lines: - if line.startswith('def '): - break - start_line_num += 1 - if python_version == 'python2': - src_lines[start_line_num] = 'def ' + component_func.__name__ + '(' + ', '.join((inspect.getfullargspec(component_func).args)) + '):' - dedecorated_component_src = '\n'.join(src_lines[start_line_num:]) - - complete_component_code = dedecorated_component_src + '\n' + wrapper_code + '\n' + codegen.end() - return complete_component_code - - def _build_image_from_tarball(self, local_tarball_path, namespace, timeout): + def _build_image(self, local_tarball_path, namespace, timeout): from ._gcs_helper import GCSHelper GCSHelper.upload_gcs_file(local_tarball_path, self._gcs_path) kaniko_spec = self._generate_kaniko_spec(namespace=namespace, - arc_dockerfile_name=self._arc_dockerfile_name, + arc_dockerfile_name=self._arc_docker_filename, gcs_path=self._gcs_path, target_image=self._target_image) # Run kaniko job @@ -370,33 +359,33 @@ def build_image_from_func(self, component_func, namespace, base_image, timeout, raise ValueError('python_version has to be either python2 or python3') with tempfile.TemporaryDirectory() as local_build_dir: # Generate entrypoint and serialization python codes - local_python_filepath = os.path.join(local_build_dir, self._arc_python_filepath) + local_python_filepath = os.path.join(local_build_dir, self._arc_python_filename) logging.info('Generate entrypoint and serialization codes.') - complete_component_code = self._generate_entrypoint(component_func, python_version) + complete_component_code = _func_to_entrypoint(component_func, python_version) with open(local_python_filepath, 'w') as f: f.write(complete_component_code) + local_requirement_filepath = os.path.join(local_build_dir, self._arc_requirement_filename) + logging.info('Generate requirement file') + _dependency_to_requirements(dependency, local_requirement_filepath) + + local_docker_filepath = os.path.join(local_build_dir, self._arc_docker_filename) + _generate_dockerfile(local_docker_filepath, base_image, self._arc_python_filename, python_version, self._arc_requirement_filename) + # Prepare build files logging.info('Generate build files.') local_tarball_path = os.path.join(local_build_dir, 'docker.tmp.tar.gz') - docker_helper = DockerfileHelper(arc_dockerfile_name=self._arc_dockerfile_name) - docker_helper.prepare_docker_tarball_with_py(python_filepath=local_python_filepath, - arc_python_filename=self._arc_python_filepath, - base_image=base_image, - local_tarball_path=local_tarball_path, - python_version=python_version, - dependency=dependency) - self._build_image_from_tarball(local_tarball_path, namespace, timeout) - - def build_image_from_dockerfile(self, dockerfile_path, timeout, namespace): + self._prepare_buildfiles(local_tarball_path, local_docker_filepath, local_python_filepath, local_requirement_filepath) + self._build_image(local_tarball_path, namespace, timeout) + + def build_image_from_dockerfile(self, docker_filename, timeout, namespace): """ build_image_from_dockerfile builds an image based on the dockerfile """ with tempfile.TemporaryDirectory() as local_build_dir: # Prepare build files logging.info('Generate build files.') local_tarball_path = os.path.join(local_build_dir, 'docker.tmp.tar.gz') - docker_helper = DockerfileHelper(arc_dockerfile_name=self._arc_dockerfile_name) - docker_helper.prepare_docker_tarball(dockerfile_path, local_tarball_path=local_tarball_path) - self._build_image_from_tarball(local_tarball_path, namespace, timeout) + self._prepare_buildfiles(local_tarball_path, docker_filename=docker_filename) + self._build_image(local_tarball_path, namespace, timeout) def _configure_logger(logger): """ _configure_logger configures the logger such that the info level logs @@ -518,5 +507,5 @@ def build_docker_image(staging_gcs_path, target_image, dockerfile_path, timeout= """ _configure_logger(logging.getLogger()) builder = ImageBuilder(gcs_base=staging_gcs_path, target_image=target_image) - builder.build_image_from_dockerfile(dockerfile_path=dockerfile_path, timeout=timeout, namespace=namespace) + builder.build_image_from_dockerfile(docker_filename=dockerfile_path, timeout=timeout, namespace=namespace) logging.info('Build image complete.') diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py index a3a70e129f0f..c5b31348c6a3 100644 --- a/sdk/python/tests/compiler/component_builder_test.py +++ b/sdk/python/tests/compiler/component_builder_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.compiler._component_builder import DockerfileHelper +from kfp.compiler._component_builder import _generate_dockerfile, _dependency_to_requirements, _func_to_entrypoint from kfp.compiler._component_builder import CodeGenerator from kfp.compiler._component_builder import ImageBuilder from kfp.compiler._component_builder import VersionedDependency @@ -110,37 +110,28 @@ def test_add_python_package(self): self.assertEqual(target_requirement_payload, golden_requirement_payload) os.remove(temp_file) +def sample_component_func(a: str, b: int) -> float: + result = 3.45 + if a == "succ": + result = float(b + 5) + return result -class TestDockerfileHelper(unittest.TestCase): - - def test_wrap_files_in_tarball(self): - """ Test wrap files in a tarball """ - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - temp_file_one = os.path.join(test_data_dir, 'test_data_one.tmp') - temp_file_two = os.path.join(test_data_dir, 'test_data_two.tmp') - temp_tarball = os.path.join(test_data_dir, 'test_data.tmp.tar.gz') - with open(temp_file_one, 'w') as f: - f.write('temporary file one content') - with open(temp_file_two, 'w') as f: - f.write('temporary file two content') +def basic_decorator(name): + def wrapper(func): + return func + return wrapper - # check - docker_helper = DockerfileHelper(arc_dockerfile_name='') - docker_helper._wrap_files_in_tarball(temp_tarball, {'dockerfile':temp_file_one, 'main.py':temp_file_two}) - self.assertTrue(os.path.exists(temp_tarball)) - with tarfile.open(temp_tarball) as temp_tarball_handle: - temp_files = temp_tarball_handle.getmembers() - self.assertTrue(len(temp_files) == 2) - for temp_file in temp_files: - self.assertTrue(temp_file.name in ['dockerfile', 'main.py']) +@basic_decorator(name='component_sample') +def sample_component_func_two(a: str, b: int) -> float: + result = 3.45 + if a == 'succ': + result = float(b + 5) + return result - # clean up - os.remove(temp_file_one) - os.remove(temp_file_two) - os.remove(temp_tarball) +def sample_component_func_three() -> float: + return 1.0 +class TestGenerator(unittest.TestCase): def test_generate_dockerfile(self): """ Test generate dockerfile """ @@ -150,171 +141,65 @@ def test_generate_dockerfile(self): golden_dockerfile_payload_one = '''\ FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0 RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools -ADD main.py /ml/ -ENTRYPOINT ["python3", "/ml/main.py"]''' +ADD main.py /ml/main.py +ENTRYPOINT ["python3", "-u", "/ml/main.py"]''' golden_dockerfile_payload_two = '''\ FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0 RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools -ADD requirements.txt /ml/ +ADD requirements.txt /ml/requirements.txt RUN pip3 install -r /ml/requirements.txt -ADD main.py /ml/ -ENTRYPOINT ["python3", "/ml/main.py"]''' +ADD main.py /ml/main.py +ENTRYPOINT ["python3", "-u", "/ml/main.py"]''' golden_dockerfile_payload_three = '''\ FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0 RUN apt-get update -y && apt-get install --no-install-recommends -y -q python python-pip python-setuptools -ADD requirements.txt /ml/ +ADD requirements.txt /ml/requirements.txt RUN pip install -r /ml/requirements.txt -ADD main.py /ml/ -ENTRYPOINT ["python", "/ml/main.py"]''' +ADD main.py /ml/main.py +ENTRYPOINT ["python", "-u", "/ml/main.py"]''' # check - docker_helper = DockerfileHelper(arc_dockerfile_name=target_dockerfile) - docker_helper._generate_dockerfile_with_py(target_file=target_dockerfile, base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', - python_filepath='main.py', has_requirement_file=False, python_version='python3') + _generate_dockerfile(filename=target_dockerfile, base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', + entrypoint_filename='main.py', python_version='python3') with open(target_dockerfile, 'r') as f: target_dockerfile_payload = f.read() self.assertEqual(target_dockerfile_payload, golden_dockerfile_payload_one) - docker_helper._generate_dockerfile_with_py(target_file=target_dockerfile, base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', - python_filepath='main.py', has_requirement_file=True, python_version='python3') + _generate_dockerfile(filename=target_dockerfile, base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', + entrypoint_filename='main.py', python_version='python3', requirement_filename='requirements.txt') with open(target_dockerfile, 'r') as f: target_dockerfile_payload = f.read() self.assertEqual(target_dockerfile_payload, golden_dockerfile_payload_two) - docker_helper._generate_dockerfile_with_py(target_file=target_dockerfile, base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', - python_filepath='main.py', has_requirement_file=True, python_version='python2') + _generate_dockerfile(filename=target_dockerfile, base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', + entrypoint_filename='main.py', python_version='python2', requirement_filename='requirements.txt') with open(target_dockerfile, 'r') as f: target_dockerfile_payload = f.read() self.assertEqual(target_dockerfile_payload, golden_dockerfile_payload_three) - self.assertRaises(ValueError, docker_helper._generate_dockerfile_with_py, target_file=target_dockerfile, - base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', python_filepath='main.py', - has_requirement_file=True, python_version='python4') + self.assertRaises(ValueError, _generate_dockerfile, filename=target_dockerfile, + base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', entrypoint_filename='main.py', + python_version='python4', requirement_filename='requirements.txt') # clean up os.remove(target_dockerfile) - def test_prepare_docker_with_py(self): - """ Test the whole prepare docker from python function """ - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - python_filepath = os.path.join(test_data_dir, 'basic.py') - local_tarball_path = os.path.join(test_data_dir, 'test_docker.tar.gz') - - # check - docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile') - docker_helper.prepare_docker_tarball_with_py(arc_python_filename='main.py', python_filepath=python_filepath, - base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.8.0', - local_tarball_path=local_tarball_path, python_version='python3') - with tarfile.open(local_tarball_path) as temp_tarball_handle: - temp_files = temp_tarball_handle.getmembers() - self.assertTrue(len(temp_files) == 2) - for temp_file in temp_files: - self.assertTrue(temp_file.name in ['dockerfile', 'main.py']) - - # clean up - os.remove(local_tarball_path) - - def test_prepare_docker_with_py_and_dependency(self): - """ Test the whole prepare docker from python function and dependencies """ - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - python_filepath = os.path.join(test_data_dir, 'basic.py') - local_tarball_path = os.path.join(test_data_dir, 'test_docker.tar.gz') - - # check - docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile') - dependencies = { - VersionedDependency(name='tensorflow', min_version='0.10.0', max_version='0.11.0'), - VersionedDependency(name='kubernetes', min_version='0.6.0'), - } - docker_helper.prepare_docker_tarball_with_py(arc_python_filename='main.py', python_filepath=python_filepath, - base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.8.0', - local_tarball_path=local_tarball_path, python_version='python3', - dependency=dependencies) - with tarfile.open(local_tarball_path) as temp_tarball_handle: - temp_files = temp_tarball_handle.getmembers() - self.assertTrue(len(temp_files) == 3) - for temp_file in temp_files: - self.assertTrue(temp_file.name in ['dockerfile', 'main.py', 'requirements.txt']) - - # clean up - os.remove(local_tarball_path) - - def test_prepare_docker_tarball(self): - """ Test the whole prepare docker tarball """ - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - dockerfile_path = os.path.join(test_data_dir, 'component.target.dockerfile') - Path(dockerfile_path).touch() - local_tarball_path = os.path.join(test_data_dir, 'test_docker.tar.gz') - - # check - docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile') - docker_helper.prepare_docker_tarball(dockerfile_path=dockerfile_path, local_tarball_path=local_tarball_path) - with tarfile.open(local_tarball_path) as temp_tarball_handle: - temp_files = temp_tarball_handle.getmembers() - self.assertTrue(len(temp_files) == 1) - for temp_file in temp_files: - self.assertTrue(temp_file.name in ['dockerfile']) - - # clean up - os.remove(local_tarball_path) - os.remove(dockerfile_path) - -# hello function is used by the TestCodeGenerator to verify the auto generated python function -def hello(): - print("hello") - -class TestCodeGenerator(unittest.TestCase): - def test_codegen(self): - """ Test code generator a function""" - codegen = CodeGenerator(indentation=' ') - codegen.begin() - codegen.writeline('def hello():') - codegen.indent() - codegen.writeline('print("hello")') - generated_codes = codegen.end() - self.assertEqual(generated_codes, inspect.getsource(hello)) - -def sample_component_func(a: str, b: int) -> float: - result = 3.45 - if a == "succ": - result = float(b + 5) - return result - -def basic_decorator(name): - def wrapper(func): - return func - return wrapper - -@basic_decorator(name='component_sample') -def sample_component_func_two(a: str, b: int) -> float: - result = 3.45 - if a == 'succ': - result = float(b + 5) - return result - -def sample_component_func_three() -> float: - return 1.0 - -class TestImageBuild(unittest.TestCase): - - def test_generate_kaniko_yaml(self): - """ Test generating the kaniko job yaml """ - + def test_generate_requirement(self): # prepare test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') + temp_file = os.path.join(test_data_dir, 'test_requirements.tmp') - # check - builder = ImageBuilder(gcs_base=GCS_BASE, target_image='') - generated_yaml = builder._generate_kaniko_spec(namespace='default', arc_dockerfile_name='dockerfile', - gcs_path='gs://mlpipeline/kaniko_build.tar.gz', target_image='gcr.io/mlpipeline/kaniko_image:latest') - with open(os.path.join(test_data_dir, 'kaniko.basic.yaml'), 'r') as f: - golden = yaml.safe_load(f) - - self.assertEqual(golden, generated_yaml) + dependencies = [ + VersionedDependency(name='tensorflow', min_version='0.10.0', max_version='0.11.0'), + VersionedDependency(name='kubernetes', min_version='0.6.0'), + ] + _dependency_to_requirements(dependencies, filename=temp_file) + golden_payload = '''\ +tensorflow >= 0.10.0, <= 0.11.0 +kubernetes >= 0.6.0 +''' + with open(temp_file, 'r') as f: + target_payload = f.read() + self.assertEqual(target_payload, golden_payload) + os.remove(temp_file) def test_generate_entrypoint(self): """ Test entrypoint generation """ @@ -323,8 +208,7 @@ def test_generate_entrypoint(self): test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') # check - builder = ImageBuilder(gcs_base=GCS_BASE, target_image='') - generated_codes = builder._generate_entrypoint(component_func=sample_component_func) + generated_codes = _func_to_entrypoint(component_func=sample_component_func) golden = '''\ def sample_component_func(a: str, b: int) -> float: result = 3.45 @@ -351,7 +235,7 @@ def wrapper_sample_component_func(a,b,_output_file): ''' self.assertEqual(golden, generated_codes) - generated_codes = builder._generate_entrypoint(component_func=sample_component_func_two) + generated_codes = _func_to_entrypoint(component_func=sample_component_func_two) golden = '''\ def sample_component_func_two(a: str, b: int) -> float: result = 3.45 @@ -378,7 +262,7 @@ def wrapper_sample_component_func_two(a,b,_output_file): ''' self.assertEqual(golden, generated_codes) - generated_codes = builder._generate_entrypoint(component_func=sample_component_func_three) + generated_codes = _func_to_entrypoint(component_func=sample_component_func_three) golden = '''\ def sample_component_func_three() -> float: return 1.0 @@ -407,8 +291,7 @@ def test_generate_entrypoint_python2(self): test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') # check - builder = ImageBuilder(gcs_base=GCS_BASE, target_image='') - generated_codes = builder._generate_entrypoint(component_func=sample_component_func_two, python_version='python2') + generated_codes = _func_to_entrypoint(component_func=sample_component_func_two, python_version='python2') golden = '''\ def sample_component_func_two(a, b): result = 3.45 @@ -433,4 +316,64 @@ def wrapper_sample_component_func_two(a,b,_output_file): if __name__ == "__main__": wrapper_sample_component_func_two(**args) ''' - self.assertEqual(golden, generated_codes) \ No newline at end of file + self.assertEqual(golden, generated_codes) + +# hello function is used by the TestCodeGenerator to verify the auto generated python function +def hello(): + print("hello") + +class TestCodeGenerator(unittest.TestCase): + def test_codegen(self): + """ Test code generator a function""" + codegen = CodeGenerator(indentation=' ') + codegen.begin() + codegen.writeline('def hello():') + codegen.indent() + codegen.writeline('print("hello")') + generated_codes = codegen.end() + self.assertEqual(generated_codes, inspect.getsource(hello)) + +class TestImageBuild(unittest.TestCase): + + def test_wrap_files_in_tarball(self): + """ Test wrap files in a tarball """ + + # prepare + test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') + temp_file_one = os.path.join(test_data_dir, 'test_data_one.tmp') + temp_file_two = os.path.join(test_data_dir, 'test_data_two.tmp') + temp_tarball = os.path.join(test_data_dir, 'test_data.tmp.tar.gz') + with open(temp_file_one, 'w') as f: + f.write('temporary file one content') + with open(temp_file_two, 'w') as f: + f.write('temporary file two content') + + # check + builder = ImageBuilder(gcs_base=GCS_BASE, target_image='') + builder._wrap_files_in_tarball(temp_tarball, {'dockerfile':temp_file_one, 'main.py':temp_file_two}) + self.assertTrue(os.path.exists(temp_tarball)) + with tarfile.open(temp_tarball) as temp_tarball_handle: + temp_files = temp_tarball_handle.getmembers() + self.assertTrue(len(temp_files) == 2) + for temp_file in temp_files: + self.assertTrue(temp_file.name in ['dockerfile', 'main.py']) + + # clean up + os.remove(temp_file_one) + os.remove(temp_file_two) + os.remove(temp_tarball) + + def test_generate_kaniko_yaml(self): + """ Test generating the kaniko job yaml """ + + # prepare + test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') + + # check + builder = ImageBuilder(gcs_base=GCS_BASE, target_image='') + generated_yaml = builder._generate_kaniko_spec(namespace='default', arc_dockerfile_name='dockerfile', + gcs_path='gs://mlpipeline/kaniko_build.tar.gz', target_image='gcr.io/mlpipeline/kaniko_image:latest') + with open(os.path.join(test_data_dir, 'kaniko.basic.yaml'), 'r') as f: + golden = yaml.safe_load(f) + + self.assertEqual(golden, generated_yaml) \ No newline at end of file