From 618b559391a225a6d821141047d6f7438e6f64a1 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 11 Jan 2019 00:33:12 -0800 Subject: [PATCH 01/15] SDK/Components - Added JsonSchema spec for the component format Added the schema outline --- .../structures/components.json_schema.json | 402 ++++++++++++++++++ .../components.json_schema.outline.yaml | 77 ++++ 2 files changed, 479 insertions(+) create mode 100644 sdk/python/kfp/components/structures/components.json_schema.json create mode 100644 sdk/python/kfp/components/structures/components.json_schema.outline.yaml diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json new file mode 100644 index 00000000000..9252979fd6b --- /dev/null +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -0,0 +1,402 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$id": "http://kubeflow.org/pipelines/components.json_schema.json", + + "$ref": "#/definitions/PipelineRunSpec", + + "definitions": { + "PrimitiveTypes": { + "oneOf": [ + {"type": "string"}, + {"type": "integer"}, + {"type": "number"}, + {"type": "boolean"} + ] + }, + + "TypeType": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"$ref": "#/definitions/TypeType"}}, + {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeType"}} + ] + }, + + "InputSpec": { + "description": "Describes the component input specification", + "type": "object", + "properties": { + "name": {"type": "string"}, + "type": {"$ref": "#/definitions/TypeType"}, + "description": {"type": "string"}, + "default": {"$ref": "#/definitions/PrimitiveTypes"}, + "optional": {"type": "boolean", "default": false} + }, + "additionalProperties": false + }, + + "OutputSpec": { + "description": "Describes the component output specification", + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"}, + "type": {"$ref": "#/definitions/TypeType"}, + "description": {"type": "string"} + }, + "additionalProperties": false + }, + + "InputValuePlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by the input argument value.", + "type": "object", + "required": ["inputValue"], + "properties": { + "inputValue" : { + "description": "Name of the input.", + "type": "string" + } + }, + "additionalProperties": false + }, + + "InputPathPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file containing the input argument value.", + "type": "object", + "required": ["inputPath"], + "properties": { + "inputPath" : { + "description": "Name of the input.", + "type": "string" + } + }, + "additionalProperties": false + }, + + "OutputPathPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file where the program should write its output data.", + "type": "object", + "required": ["outputPath"], + "properties": { + "outputPath" : { + "description": "Name of the output.", + "type": "string" + } + }, + "additionalProperties": false + }, + + "CommandlineArgumentType": { + "oneOf": [ + {"$ref": "#/definitions/PrimitiveTypes"}, + {"$ref": "#/definitions/InputValuePlaceholder"}, + {"$ref": "#/definitions/InputPathPlaceholder"}, + {"$ref": "#/definitions/OutputPathPlaceholder"}, + {"$ref": "#/definitions/ConcatPlaceholder"}, + {"$ref": "#/definitions/IfPlaceholder"} + ] + }, + + "CommandlineArgumentOrArrayType": { + "oneOf": [ + {"$ref": "#/definitions/CommandlineArgumentType"}, + {"type": "array", "items": {"$ref": "#/definitions/CommandlineArgumentType"}} + ] + }, + + "ConcatPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by the concatenated values of its items.", + "type": "object", + "required": ["concat"], + "properties": { + "concat" : { + "description": "Items to concatenate", + "type": "array", + "items": {"$ref": "#/definitions/CommandlineArgumentType"} + } + }, + "additionalProperties": false + }, + + "IsPresentPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", + "type": "object", + "properties": { + "isPresent": { + "description": "Name of the input.", + "type": "string" + } + }, + "additionalProperties": false + }, + + "IfConditionArgumentType": { + "oneOf": [ + {"$ref": "#/definitions/IsPresentPlaceholder"}, + {"type": "boolean"}, + {"type": "string"}, + {"$ref": "#/definitions/InputValuePlaceholder"} + ] + }, + + "IfPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", + "type": "object", + "required": ["if"], + "properties": { + "if" : { + "type": "object", + "required": ["cond", "then"], + "properties": { + "cond": {"$ref": "#/definitions/IfConditionArgumentType"}, + "then": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"}, + "else": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"} + } + } + } + }, + + "ContainerSpec": { + "type": "object", + "required": ["image"], + "properties": { + "image": { + "description": "Docker image name.", + "type": "string" + }, + "command": { + "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided.", + "type": "array", + "items": {"$ref": "#/definitions/CommandlineArgumentType"} + }, + "args": { + "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided.", + "type": "array", + "items": {"$ref": "#/definitions/CommandlineArgumentType"} + }, + "env": { + "description": "List of environment variables to set in the container.", + "type": "object", + "additionalProperties": {"type": "string"} + }, + "unconfigurableOutputPaths": { + "description": "Legacy. Deprecated. Can be used to specify local output file paths for containers that do not allow setting the path of some output using command-line.", + "type": "object", + "additionalProperties": {"type": "string"} + } + }, + "additionalProperties": false + }, + + "ContainerImplementation": { + "description": "Represents the container component implementation.", + "type": "object", + "required": ["container"], + "properties": { + "container": {"$ref": "#/definitions/ContainerSpec"} + } + }, + + "ImplementationType": { + "oneOf": [ + {"$ref": "#/definitions/ContainerImplementation"}, + {"$ref": "#/definitions/GraphImplementation"} + ] + }, + + "SourceSpec": { + "description": "Specifies the location of the component source code.", + "type": "object", + "properties": { + "url": {"type": "string"} + }, + "additionalProperties": false + }, + + "ComponentSpec": { + "description": "Component specification. Describes the metadata (name, description, source), the interface (inputs and outputs) and the implementation of the component.", + "type": "object", + "required": ["implementation"], + "properties": { + "name": {"type": "string"}, + "description": {"type": "string"}, + "source": {"$ref": "#/definitions/SourceSpec"}, + "inputs": {"type": "array", "items": {"$ref": "#/definitions/InputSpec"}}, + "outputs": {"type": "array", "items": {"$ref": "#/definitions/OutputSpec"}}, + "implementation": {"$ref": "#/definitions/ImplementationType"}, + "schemaVersion": {"type": "string", "default": "kubeflow.org/pipelines/component/v1"} + }, + "additionalProperties": false + }, + + "ComponentReference": { + "description": "Component reference. Contains information that can be used to locate and load a component by name, digest or URL", + "type": "object", + "properties": { + "name": {"type": "string"}, + "digest": {"type": "string"}, + "tag": {"type": "string"}, + "url": {"type": "string"}, + "spec": {"$ref": "#/definitions/ComponentSpec"} + }, + "additionalProperties": false + }, + + "GraphInputArgument": { + "description": "Represents the component argument value that comes from the graph component input.", + "type": "object", + "required": ["graphInput"], + "properties": { + "graphInput": { + "description": "Input name", + "type": "string" + } + }, + "additionalProperties": false + }, + + "TaskOutputArgument": { + "description": "Represents the component argument value that comes from the output of a sibling task.", + "type": "object", + "required": ["taskOutput"], + "properties": { + "taskOutput": { + "description": "References the output of a sibling task.", + "type": "object", + "required": ["taskId", "outputName"], + "properties": { + "taskId": {"type": "string"}, + "outputName": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "ArgumentType": { + "oneOf": [ + {"$ref": "#/definitions/PrimitiveTypes"}, + {"$ref": "#/definitions/GraphInputArgument"}, + {"$ref": "#/definitions/TaskOutputArgument"} + ] + }, + + "TwoArgumentOperands": { + "description": "Pair of operands for a binary operation.", + "type": "object", + "required": ["op1", "op2"], + "properties": { + "op1": {"$ref": "#/definitions/ArgumentType"}, + "op2": {"$ref": "#/definitions/ArgumentType"} + }, + "additionalProperties": false + }, + + "TwoLogicalOperands": { + "description": "Pair of operands for a binary logical operation.", + "type": "object", + "required": ["op1", "op2"], + "properties": { + "op1": {"$ref": "#/definitions/PredicateType"}, + "op2": {"$ref": "#/definitions/PredicateType"} + }, + "additionalProperties": false + }, + + "PredicateType": { + "oneOf": [ + {"type": "object", "required": ["=="], "properties": {"==": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": ["!="], "properties": {"!=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": [">"], "properties": {">": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": [">="], "properties": {">=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": ["<"], "properties": {"<": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": ["<="], "properties": {"<=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + + {"type": "object", "required": ["and"], "properties": {"and": {"$ref": "#/definitions/TwoLogicalOperands"}}}, + {"type": "object", "required": ["or"], "properties": {"or": {"$ref": "#/definitions/TwoLogicalOperands"}}}, + + {"type": "object", "required": ["not"], "properties": {"not": {"$ref": "#/definitions/PredicateType"}}} + ] + }, + + "ObjectMetaArgoSubset": { + "description": "Subset of io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta structure supported by Argo", + "type": "object", + "properties": { + "annotations": {"type": "object", "additionalItems": {"type": "string"}}, + "labels": {"type": "object", "additionalItems": {"type": "string"}} + }, + "additionalProperties": false + }, + + "PodSpecArgoSubset": { + "description": "Subset of Kubernetes PodSpec structure supported by Argo", + "type": "object", + "properties": { + "activeDeadlineSeconds": {"type": "integer"}, + "affinity": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity"}, + "tolerations": {"type": "array", "items": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration"}}, + "volumes": {"type": "array", "items": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Volume"}}, + "nodeSelector": {"type": "object", "additionalItems": {"type": "string"}} + }, + "additionalProperties": false + }, + + "PodArgoSubset": { + "description": "Subset of io.k8s.api.core.v1.Pod structure supported by Argo", + "type": "object", + "properties": { + "metadata": {"$ref": "#/definitions/ObjectMetaArgoSubset"}, + "spec": {"$ref": "#/definitions/PodSpecArgoSubset"} + }, + "additionalProperties": false + }, + + "TaskSpec": { + "description": "'Task specification. Task is a configured component - a component supplied with arguments and other applied configuration changes.", + "type": "object", + "required": ["componentRef"], + "properties": { + "componentRef": {"$ref": "#/definitions/ComponentReference"}, + "arguments": {"type": "object", "additionalProperties": {"$ref": "#/definitions/ArgumentType"}}, + "isEnabled": {"$ref": "#/definitions/PredicateType"}, + "k8sContainerOptions": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, + "k8sPodOptions": {"$ref": "#/definitions/PodArgoSubset"} + }, + "additionalProperties": false + }, + + "GraphSpec": { + "description": "Describes the graph component implementation. It represents a graph of component tasks connected to the upstream sources of data using the argument specifications. It also describes the sources of graph output values.", + "type": "object", + "required": ["tasks"], + "properties": { + "tasks": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskSpec"}}, + "outputValues": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskOutputArgument"}} + }, + "additionalProperties": false + }, + + "GraphImplementation": { + "description": "Represents the graph component implementation.", + "type": "object", + "required": ["graph"], + "properties": { + "graph": {"$ref": "#/definitions/GraphSpec"} + }, + "additionalProperties": false + }, + + "PipelineRunSpec": { + "description": "The object that can be sent to the backend to start a new Run.", + "type": "object", + "required": ["rootTask"], + "properties": { + "rootTask": {"$ref": "#/definitions/TaskSpec"}, + "onExitTask": {"$ref": "#/definitions/TaskSpec"} + }, + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml new file mode 100644 index 00000000000..3a0d6e50859 --- /dev/null +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -0,0 +1,77 @@ +: + name: string + description: string + source: #SourceSpec + url: string + schemaVersion: string + inputs: #InputSpec[] + : + name: string + type: TypeType + description: string + default: PrimitiveTypes + optional: boolean + outputs: #OutputSpec[] + : + name: string + type: TypeType + description: string + implementation: #ImplementationType + : + : + container: #ContainerSpec + image: string + command: #CommandlineArgumentType + : + : + : string, integer, number, boolean + : + inputValue: string + : + inputPath: string + : + outputPath: string + : + concat: CommandlineArgumentType[] + : + if: + cond: IfConditionArgumentType + then: CommandlineArgumentType[] + else: CommandlineArgumentType[] + args: CommandlineArgumentType + env: Map[str, str] + unconfigurableOutputPaths: Map[str, str] + : + graph: #GraphSpec + outputValues: Map str -> TaskOutputArgument + tasks: #Map str -> TaskSpec + TaskSpec>: + componentRef: #ComponentReference + name: string + digest: string + tag: string + url: string + spec: ComponentSpec + arguments: #Map str -> ArgumentType + ArgumentType>: + : + : + oneOf: string, integer, number, boolean + : + graphInput: string + : + taskOutput: + taskId: string + outputName: string + isEnabled: PredicateType + k8sContainerOptions: io.k8s.api.core.v1.Container + k8sPodOptions: #PodArgoSubset + metadata: #ObjectMetaArgoSubset + annotations: Map str -> str + labels: Map str -> str + spec: #PodSpecArgoSubset + activeDeadlineSeconds: integer + affinity: io.k8s.api.core.v1.Affinity + tolerations: io.k8s.api.core.v1.Toleration[] + volumes: io.k8s.api.core.v1.Volume[] + nodeSelector: Map str -> str From f8ada559116486f175796495c084ce593cc8ae7c Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:03:08 -0800 Subject: [PATCH 02/15] Fixed missing "required" --- sdk/python/kfp/components/structures/components.json_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 9252979fd6b..c81330a6623 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -25,6 +25,7 @@ "InputSpec": { "description": "Describes the component input specification", "type": "object", + "required": ["name"], "properties": { "name": {"type": "string"}, "type": {"$ref": "#/definitions/TypeType"}, From feae5df66b52b3c94004aefbd180c4d5256e70d5 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:06:20 -0800 Subject: [PATCH 03/15] Replaced PrimitiveTypes with just string --- .../structures/components.json_schema.json | 15 +++------------ .../components.json_schema.outline.yaml | 8 +++----- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index c81330a6623..cd1a255d459 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -5,15 +5,6 @@ "$ref": "#/definitions/PipelineRunSpec", "definitions": { - "PrimitiveTypes": { - "oneOf": [ - {"type": "string"}, - {"type": "integer"}, - {"type": "number"}, - {"type": "boolean"} - ] - }, - "TypeType": { "oneOf": [ {"type": "string"}, @@ -30,7 +21,7 @@ "name": {"type": "string"}, "type": {"$ref": "#/definitions/TypeType"}, "description": {"type": "string"}, - "default": {"$ref": "#/definitions/PrimitiveTypes"}, + "default": {"type": "string"}, "optional": {"type": "boolean", "default": false} }, "additionalProperties": false @@ -89,7 +80,7 @@ "CommandlineArgumentType": { "oneOf": [ - {"$ref": "#/definitions/PrimitiveTypes"}, + {"type": "string"}, {"$ref": "#/definitions/InputValuePlaceholder"}, {"$ref": "#/definitions/InputPathPlaceholder"}, {"$ref": "#/definitions/OutputPathPlaceholder"}, @@ -277,7 +268,7 @@ "ArgumentType": { "oneOf": [ - {"$ref": "#/definitions/PrimitiveTypes"}, + {"type": "string"}, {"$ref": "#/definitions/GraphInputArgument"}, {"$ref": "#/definitions/TaskOutputArgument"} ] diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index 3a0d6e50859..fa36d6154ee 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -9,7 +9,7 @@ name: string type: TypeType description: string - default: PrimitiveTypes + default: string optional: boolean outputs: #OutputSpec[] : @@ -23,8 +23,7 @@ image: string command: #CommandlineArgumentType : - : - : string, integer, number, boolean + string: : inputValue: string : @@ -55,8 +54,7 @@ arguments: #Map str -> ArgumentType ArgumentType>: : - : - oneOf: string, integer, number, boolean + string: : graphInput: string : From cdfa0ff5abab84695a7a6faf8e45ebda9f3bcc19 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:27:26 -0800 Subject: [PATCH 04/15] Renamed CommandlineArgumentType to StringOrPlaceholder Made more ContainerSpec properties support placeholders --- .../structures/components.json_schema.json | 19 ++++++------------- .../components.json_schema.outline.yaml | 14 +++++++------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index cd1a255d459..be52a5dcf48 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -78,7 +78,7 @@ "additionalProperties": false }, - "CommandlineArgumentType": { + "StringOrPlaceholder": { "oneOf": [ {"type": "string"}, {"$ref": "#/definitions/InputValuePlaceholder"}, @@ -89,13 +89,6 @@ ] }, - "CommandlineArgumentOrArrayType": { - "oneOf": [ - {"$ref": "#/definitions/CommandlineArgumentType"}, - {"type": "array", "items": {"$ref": "#/definitions/CommandlineArgumentType"}} - ] - }, - "ConcatPlaceholder": { "description": "Represents the command-line argument placeholder that will be replaced at run-time by the concatenated values of its items.", "type": "object", @@ -104,7 +97,7 @@ "concat" : { "description": "Items to concatenate", "type": "array", - "items": {"$ref": "#/definitions/CommandlineArgumentType"} + "items": {"$ref": "#/definitions/StringOrPlaceholder"} } }, "additionalProperties": false @@ -154,22 +147,22 @@ "properties": { "image": { "description": "Docker image name.", - "type": "string" + "$ref": "#/definitions/StringOrPlaceholder" }, "command": { "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided.", "type": "array", - "items": {"$ref": "#/definitions/CommandlineArgumentType"} + "items": {"$ref": "#/definitions/StringOrPlaceholder"} }, "args": { "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided.", "type": "array", - "items": {"$ref": "#/definitions/CommandlineArgumentType"} + "items": {"$ref": "#/definitions/StringOrPlaceholder"} }, "env": { "description": "List of environment variables to set in the container.", "type": "object", - "additionalProperties": {"type": "string"} + "additionalProperties": {"$ref": "#/definitions/StringOrPlaceholder"} }, "unconfigurableOutputPaths": { "description": "Legacy. Deprecated. Can be used to specify local output file paths for containers that do not allow setting the path of some output using command-line.", diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index fa36d6154ee..6e4219f60ac 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -20,8 +20,8 @@ : : container: #ContainerSpec - image: string - command: #CommandlineArgumentType + image: StringOrPlaceholder + command: #StringOrPlaceholder[] : string: : @@ -31,14 +31,14 @@ : outputPath: string : - concat: CommandlineArgumentType[] + concat: StringOrPlaceholder[] : if: cond: IfConditionArgumentType - then: CommandlineArgumentType[] - else: CommandlineArgumentType[] - args: CommandlineArgumentType - env: Map[str, str] + then: StringOrPlaceholder[] + else: StringOrPlaceholder[] + args: StringOrPlaceholder[] + env: Map[str, StringOrPlaceholder] unconfigurableOutputPaths: Map[str, str] : graph: #GraphSpec From 4bd6c201d762d5527c792fce7a153c4c28f5315f Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:12:42 -0800 Subject: [PATCH 05/15] Removed support for type inheritance and generic types as requested by Ning and Ajay --- .../kfp/components/structures/components.json_schema.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index be52a5dcf48..ae648eec10d 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -8,8 +8,7 @@ "TypeType": { "oneOf": [ {"type": "string"}, - {"type": "array", "items": {"$ref": "#/definitions/TypeType"}}, - {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeType"}} + {"type": "object", "additionalProperties": {"type": "string"}} ] }, From c8677d3ba46f1c27e1ae70b4757a5eb471930263 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:17:51 -0800 Subject: [PATCH 06/15] Some people are scared of graphs/pipelines - removing them --- .../structures/components.json_schema.json | 38 +----------- .../components.json_schema.outline.yaml | 61 +++++++++---------- 2 files changed, 29 insertions(+), 70 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index ae648eec10d..6cac0034bf5 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -183,8 +183,7 @@ "ImplementationType": { "oneOf": [ - {"$ref": "#/definitions/ContainerImplementation"}, - {"$ref": "#/definitions/GraphImplementation"} + {"$ref": "#/definitions/ContainerImplementation"} ] }, @@ -226,19 +225,6 @@ "additionalProperties": false }, - "GraphInputArgument": { - "description": "Represents the component argument value that comes from the graph component input.", - "type": "object", - "required": ["graphInput"], - "properties": { - "graphInput": { - "description": "Input name", - "type": "string" - } - }, - "additionalProperties": false - }, - "TaskOutputArgument": { "description": "Represents the component argument value that comes from the output of a sibling task.", "type": "object", @@ -261,7 +247,6 @@ "ArgumentType": { "oneOf": [ {"type": "string"}, - {"$ref": "#/definitions/GraphInputArgument"}, {"$ref": "#/definitions/TaskOutputArgument"} ] }, @@ -351,27 +336,6 @@ "additionalProperties": false }, - "GraphSpec": { - "description": "Describes the graph component implementation. It represents a graph of component tasks connected to the upstream sources of data using the argument specifications. It also describes the sources of graph output values.", - "type": "object", - "required": ["tasks"], - "properties": { - "tasks": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskSpec"}}, - "outputValues": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskOutputArgument"}} - }, - "additionalProperties": false - }, - - "GraphImplementation": { - "description": "Represents the graph component implementation.", - "type": "object", - "required": ["graph"], - "properties": { - "graph": {"$ref": "#/definitions/GraphSpec"} - }, - "additionalProperties": false - }, - "PipelineRunSpec": { "description": "The object that can be sent to the backend to start a new Run.", "type": "object", diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index 6e4219f60ac..87f852e0f07 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -40,36 +40,31 @@ args: StringOrPlaceholder[] env: Map[str, StringOrPlaceholder] unconfigurableOutputPaths: Map[str, str] - : - graph: #GraphSpec - outputValues: Map str -> TaskOutputArgument - tasks: #Map str -> TaskSpec - TaskSpec>: - componentRef: #ComponentReference - name: string - digest: string - tag: string - url: string - spec: ComponentSpec - arguments: #Map str -> ArgumentType - ArgumentType>: - : - string: - : - graphInput: string - : - taskOutput: - taskId: string - outputName: string - isEnabled: PredicateType - k8sContainerOptions: io.k8s.api.core.v1.Container - k8sPodOptions: #PodArgoSubset - metadata: #ObjectMetaArgoSubset - annotations: Map str -> str - labels: Map str -> str - spec: #PodSpecArgoSubset - activeDeadlineSeconds: integer - affinity: io.k8s.api.core.v1.Affinity - tolerations: io.k8s.api.core.v1.Toleration[] - volumes: io.k8s.api.core.v1.Volume[] - nodeSelector: Map str -> str + +: + componentRef: #ComponentReference + name: string + digest: string + tag: string + url: string + spec: ComponentSpec + arguments: #Map str -> ArgumentType + ArgumentType>: + : + string: + : + taskOutput: + taskId: string + outputName: string + isEnabled: PredicateType + k8sContainerOptions: io.k8s.api.core.v1.Container + k8sPodOptions: #PodArgoSubset + metadata: #ObjectMetaArgoSubset + annotations: Map str -> str + labels: Map str -> str + spec: #PodSpecArgoSubset + activeDeadlineSeconds: integer + affinity: io.k8s.api.core.v1.Affinity + tolerations: io.k8s.api.core.v1.Toleration[] + volumes: io.k8s.api.core.v1.Volume[] + nodeSelector: Map str -> str From 8c47f289697f775a4554b316f8ce2007b251b8a8 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:22:25 -0800 Subject: [PATCH 07/15] Some people do not like optional inputs and conditionals - removing them Sorry, Yasser and Bradley. --- .../structures/components.json_schema.json | 44 +------------------ .../components.json_schema.outline.yaml | 6 --- 2 files changed, 2 insertions(+), 48 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 6cac0034bf5..406176ff2bb 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -20,8 +20,7 @@ "name": {"type": "string"}, "type": {"$ref": "#/definitions/TypeType"}, "description": {"type": "string"}, - "default": {"type": "string"}, - "optional": {"type": "boolean", "default": false} + "default": {"type": "string"} }, "additionalProperties": false }, @@ -83,8 +82,7 @@ {"$ref": "#/definitions/InputValuePlaceholder"}, {"$ref": "#/definitions/InputPathPlaceholder"}, {"$ref": "#/definitions/OutputPathPlaceholder"}, - {"$ref": "#/definitions/ConcatPlaceholder"}, - {"$ref": "#/definitions/IfPlaceholder"} + {"$ref": "#/definitions/ConcatPlaceholder"} ] }, @@ -102,44 +100,6 @@ "additionalProperties": false }, - "IsPresentPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", - "type": "object", - "properties": { - "isPresent": { - "description": "Name of the input.", - "type": "string" - } - }, - "additionalProperties": false - }, - - "IfConditionArgumentType": { - "oneOf": [ - {"$ref": "#/definitions/IsPresentPlaceholder"}, - {"type": "boolean"}, - {"type": "string"}, - {"$ref": "#/definitions/InputValuePlaceholder"} - ] - }, - - "IfPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", - "type": "object", - "required": ["if"], - "properties": { - "if" : { - "type": "object", - "required": ["cond", "then"], - "properties": { - "cond": {"$ref": "#/definitions/IfConditionArgumentType"}, - "then": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"}, - "else": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"} - } - } - } - }, - "ContainerSpec": { "type": "object", "required": ["image"], diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index 87f852e0f07..d86f62e094c 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -10,7 +10,6 @@ type: TypeType description: string default: string - optional: boolean outputs: #OutputSpec[] : name: string @@ -32,11 +31,6 @@ outputPath: string : concat: StringOrPlaceholder[] - : - if: - cond: IfConditionArgumentType - then: StringOrPlaceholder[] - else: StringOrPlaceholder[] args: StringOrPlaceholder[] env: Map[str, StringOrPlaceholder] unconfigurableOutputPaths: Map[str, str] From b7fb1b7f931af625bd7090fc30024cb15cf8d6ee Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 5 Feb 2019 23:36:15 -0800 Subject: [PATCH 08/15] Some people might be scared of predicates or conditional execution - removing them Sure, Argo and DSL supports it, but some people care more about the spec file size even when that means dropping already supported features. Sorry, Bradley and Riley. --- .../structures/components.json_schema.json | 39 ------------------- .../components.json_schema.outline.yaml | 1 - 2 files changed, 40 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 406176ff2bb..2fc7340e724 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -211,44 +211,6 @@ ] }, - "TwoArgumentOperands": { - "description": "Pair of operands for a binary operation.", - "type": "object", - "required": ["op1", "op2"], - "properties": { - "op1": {"$ref": "#/definitions/ArgumentType"}, - "op2": {"$ref": "#/definitions/ArgumentType"} - }, - "additionalProperties": false - }, - - "TwoLogicalOperands": { - "description": "Pair of operands for a binary logical operation.", - "type": "object", - "required": ["op1", "op2"], - "properties": { - "op1": {"$ref": "#/definitions/PredicateType"}, - "op2": {"$ref": "#/definitions/PredicateType"} - }, - "additionalProperties": false - }, - - "PredicateType": { - "oneOf": [ - {"type": "object", "required": ["=="], "properties": {"==": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": ["!="], "properties": {"!=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": [">"], "properties": {">": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": [">="], "properties": {">=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": ["<"], "properties": {"<": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": ["<="], "properties": {"<=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - - {"type": "object", "required": ["and"], "properties": {"and": {"$ref": "#/definitions/TwoLogicalOperands"}}}, - {"type": "object", "required": ["or"], "properties": {"or": {"$ref": "#/definitions/TwoLogicalOperands"}}}, - - {"type": "object", "required": ["not"], "properties": {"not": {"$ref": "#/definitions/PredicateType"}}} - ] - }, - "ObjectMetaArgoSubset": { "description": "Subset of io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta structure supported by Argo", "type": "object", @@ -289,7 +251,6 @@ "properties": { "componentRef": {"$ref": "#/definitions/ComponentReference"}, "arguments": {"type": "object", "additionalProperties": {"$ref": "#/definitions/ArgumentType"}}, - "isEnabled": {"$ref": "#/definitions/PredicateType"}, "k8sContainerOptions": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, "k8sPodOptions": {"$ref": "#/definitions/PodArgoSubset"} }, diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index d86f62e094c..ba615410166 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -50,7 +50,6 @@ taskOutput: taskId: string outputName: string - isEnabled: PredicateType k8sContainerOptions: io.k8s.api.core.v1.Container k8sPodOptions: #PodArgoSubset metadata: #ObjectMetaArgoSubset From 24b5452c0164c9eadc258c13b8bd28e04008db12 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:01:11 -0800 Subject: [PATCH 09/15] Reverting the last 4 commits Making those big compromises did not have any noticeable effect on people asking for them. --- .../structures/components.json_schema.json | 126 +++++++++++++++++- .../components.json_schema.outline.yaml | 66 +++++---- 2 files changed, 160 insertions(+), 32 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 2fc7340e724..4d850831d17 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -8,7 +8,8 @@ "TypeType": { "oneOf": [ {"type": "string"}, - {"type": "object", "additionalProperties": {"type": "string"}} + {"type": "array", "items": {"$ref": "#/definitions/TypeType"}}, + {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeType"}} ] }, @@ -20,7 +21,8 @@ "name": {"type": "string"}, "type": {"$ref": "#/definitions/TypeType"}, "description": {"type": "string"}, - "default": {"type": "string"} + "default": {"type": "string"}, + "optional": {"type": "boolean", "default": false} }, "additionalProperties": false }, @@ -82,7 +84,8 @@ {"$ref": "#/definitions/InputValuePlaceholder"}, {"$ref": "#/definitions/InputPathPlaceholder"}, {"$ref": "#/definitions/OutputPathPlaceholder"}, - {"$ref": "#/definitions/ConcatPlaceholder"} + {"$ref": "#/definitions/ConcatPlaceholder"}, + {"$ref": "#/definitions/IfPlaceholder"} ] }, @@ -100,6 +103,44 @@ "additionalProperties": false }, + "IsPresentPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", + "type": "object", + "properties": { + "isPresent": { + "description": "Name of the input.", + "type": "string" + } + }, + "additionalProperties": false + }, + + "IfConditionArgumentType": { + "oneOf": [ + {"$ref": "#/definitions/IsPresentPlaceholder"}, + {"type": "boolean"}, + {"type": "string"}, + {"$ref": "#/definitions/InputValuePlaceholder"} + ] + }, + + "IfPlaceholder": { + "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", + "type": "object", + "required": ["if"], + "properties": { + "if" : { + "type": "object", + "required": ["cond", "then"], + "properties": { + "cond": {"$ref": "#/definitions/IfConditionArgumentType"}, + "then": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"}, + "else": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"} + } + } + } + }, + "ContainerSpec": { "type": "object", "required": ["image"], @@ -143,7 +184,8 @@ "ImplementationType": { "oneOf": [ - {"$ref": "#/definitions/ContainerImplementation"} + {"$ref": "#/definitions/ContainerImplementation"}, + {"$ref": "#/definitions/GraphImplementation"} ] }, @@ -185,6 +227,19 @@ "additionalProperties": false }, + "GraphInputArgument": { + "description": "Represents the component argument value that comes from the graph component input.", + "type": "object", + "required": ["graphInput"], + "properties": { + "graphInput": { + "description": "Input name", + "type": "string" + } + }, + "additionalProperties": false + }, + "TaskOutputArgument": { "description": "Represents the component argument value that comes from the output of a sibling task.", "type": "object", @@ -207,10 +262,49 @@ "ArgumentType": { "oneOf": [ {"type": "string"}, + {"$ref": "#/definitions/GraphInputArgument"}, {"$ref": "#/definitions/TaskOutputArgument"} ] }, + "TwoArgumentOperands": { + "description": "Pair of operands for a binary operation.", + "type": "object", + "required": ["op1", "op2"], + "properties": { + "op1": {"$ref": "#/definitions/ArgumentType"}, + "op2": {"$ref": "#/definitions/ArgumentType"} + }, + "additionalProperties": false + }, + + "TwoLogicalOperands": { + "description": "Pair of operands for a binary logical operation.", + "type": "object", + "required": ["op1", "op2"], + "properties": { + "op1": {"$ref": "#/definitions/PredicateType"}, + "op2": {"$ref": "#/definitions/PredicateType"} + }, + "additionalProperties": false + }, + + "PredicateType": { + "oneOf": [ + {"type": "object", "required": ["=="], "properties": {"==": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": ["!="], "properties": {"!=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": [">"], "properties": {">": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": [">="], "properties": {">=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": ["<"], "properties": {"<": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + {"type": "object", "required": ["<="], "properties": {"<=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, + + {"type": "object", "required": ["and"], "properties": {"and": {"$ref": "#/definitions/TwoLogicalOperands"}}}, + {"type": "object", "required": ["or"], "properties": {"or": {"$ref": "#/definitions/TwoLogicalOperands"}}}, + + {"type": "object", "required": ["not"], "properties": {"not": {"$ref": "#/definitions/PredicateType"}}} + ] + }, + "ObjectMetaArgoSubset": { "description": "Subset of io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta structure supported by Argo", "type": "object", @@ -251,12 +345,34 @@ "properties": { "componentRef": {"$ref": "#/definitions/ComponentReference"}, "arguments": {"type": "object", "additionalProperties": {"$ref": "#/definitions/ArgumentType"}}, + "isEnabled": {"$ref": "#/definitions/PredicateType"}, "k8sContainerOptions": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, "k8sPodOptions": {"$ref": "#/definitions/PodArgoSubset"} }, "additionalProperties": false }, + "GraphSpec": { + "description": "Describes the graph component implementation. It represents a graph of component tasks connected to the upstream sources of data using the argument specifications. It also describes the sources of graph output values.", + "type": "object", + "required": ["tasks"], + "properties": { + "tasks": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskSpec"}}, + "outputValues": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskOutputArgument"}} + }, + "additionalProperties": false + }, + + "GraphImplementation": { + "description": "Represents the graph component implementation.", + "type": "object", + "required": ["graph"], + "properties": { + "graph": {"$ref": "#/definitions/GraphSpec"} + }, + "additionalProperties": false + }, + "PipelineRunSpec": { "description": "The object that can be sent to the backend to start a new Run.", "type": "object", @@ -268,4 +384,4 @@ "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index ba615410166..6e4219f60ac 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -10,6 +10,7 @@ type: TypeType description: string default: string + optional: boolean outputs: #OutputSpec[] : name: string @@ -31,33 +32,44 @@ outputPath: string : concat: StringOrPlaceholder[] + : + if: + cond: IfConditionArgumentType + then: StringOrPlaceholder[] + else: StringOrPlaceholder[] args: StringOrPlaceholder[] env: Map[str, StringOrPlaceholder] unconfigurableOutputPaths: Map[str, str] - -: - componentRef: #ComponentReference - name: string - digest: string - tag: string - url: string - spec: ComponentSpec - arguments: #Map str -> ArgumentType - ArgumentType>: - : - string: - : - taskOutput: - taskId: string - outputName: string - k8sContainerOptions: io.k8s.api.core.v1.Container - k8sPodOptions: #PodArgoSubset - metadata: #ObjectMetaArgoSubset - annotations: Map str -> str - labels: Map str -> str - spec: #PodSpecArgoSubset - activeDeadlineSeconds: integer - affinity: io.k8s.api.core.v1.Affinity - tolerations: io.k8s.api.core.v1.Toleration[] - volumes: io.k8s.api.core.v1.Volume[] - nodeSelector: Map str -> str + : + graph: #GraphSpec + outputValues: Map str -> TaskOutputArgument + tasks: #Map str -> TaskSpec + TaskSpec>: + componentRef: #ComponentReference + name: string + digest: string + tag: string + url: string + spec: ComponentSpec + arguments: #Map str -> ArgumentType + ArgumentType>: + : + string: + : + graphInput: string + : + taskOutput: + taskId: string + outputName: string + isEnabled: PredicateType + k8sContainerOptions: io.k8s.api.core.v1.Container + k8sPodOptions: #PodArgoSubset + metadata: #ObjectMetaArgoSubset + annotations: Map str -> str + labels: Map str -> str + spec: #PodSpecArgoSubset + activeDeadlineSeconds: integer + affinity: io.k8s.api.core.v1.Affinity + tolerations: io.k8s.api.core.v1.Toleration[] + volumes: io.k8s.api.core.v1.Volume[] + nodeSelector: Map str -> str From bcd6cb8334725bfbcb1283a454fdaaec25adb2b9 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:02:49 -0800 Subject: [PATCH 10/15] Removed list-style type specifications We've standardized on the map-style specification. --- sdk/python/kfp/components/structures/components.json_schema.json | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 4d850831d17..7f9a4527a31 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -8,7 +8,6 @@ "TypeType": { "oneOf": [ {"type": "string"}, - {"type": "array", "items": {"$ref": "#/definitions/TypeType"}}, {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeType"}} ] }, From 0ded4608a4eded2c64839a5f96df48e7440ac3cc Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:04:29 -0800 Subject: [PATCH 11/15] Renamed TypeType to TypeSpecType --- .../kfp/components/structures/components.json_schema.json | 8 ++++---- .../structures/components.json_schema.outline.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 7f9a4527a31..722426f3fce 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -5,10 +5,10 @@ "$ref": "#/definitions/PipelineRunSpec", "definitions": { - "TypeType": { + "TypeSpecType": { "oneOf": [ {"type": "string"}, - {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeType"}} + {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeSpecType"}} ] }, @@ -18,7 +18,7 @@ "required": ["name"], "properties": { "name": {"type": "string"}, - "type": {"$ref": "#/definitions/TypeType"}, + "type": {"$ref": "#/definitions/TypeSpecType"}, "description": {"type": "string"}, "default": {"type": "string"}, "optional": {"type": "boolean", "default": false} @@ -32,7 +32,7 @@ "required": ["name"], "properties": { "name": {"type": "string"}, - "type": {"$ref": "#/definitions/TypeType"}, + "type": {"$ref": "#/definitions/TypeSpecType"}, "description": {"type": "string"} }, "additionalProperties": false diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index 6e4219f60ac..55850f21565 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -7,14 +7,14 @@ inputs: #InputSpec[] : name: string - type: TypeType + type: TypeSpecType description: string default: string optional: boolean outputs: #OutputSpec[] : name: string - type: TypeType + type: TypeSpecType description: string implementation: #ImplementationType : From 0697a3ed220391474cc2ec8a6d0ed2a3d2a06b5f Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:09:24 -0800 Subject: [PATCH 12/15] Updated the structure of graphInput --- .../components/structures/components.json_schema.json | 9 +++++++-- .../structures/components.json_schema.outline.yaml | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 722426f3fce..28ab0e435e7 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -232,8 +232,13 @@ "required": ["graphInput"], "properties": { "graphInput": { - "description": "Input name", - "type": "string" + "description": "References the input of the graph/pipeline.", + "type": "object", + "required": ["inputName"], + "properties": { + "inputName": {"type": "string"} + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index 55850f21565..de089d7e0e1 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -56,7 +56,8 @@ : string: : - graphInput: string + graphInput: + inputName: string : taskOutput: taskId: string From ed7b6c092cf76bca11410a1e3daabfcc44af2123 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:11:12 -0800 Subject: [PATCH 13/15] Added the type attribute to taskOutput and graphInput --- .../kfp/components/structures/components.json_schema.json | 6 ++++-- .../structures/components.json_schema.outline.yaml | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 28ab0e435e7..9e87ad91b6b 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -236,7 +236,8 @@ "type": "object", "required": ["inputName"], "properties": { - "inputName": {"type": "string"} + "inputName": {"type": "string"}, + "type": {"$ref": "#/definitions/TypeSpecType"} }, "additionalProperties": false } @@ -255,7 +256,8 @@ "required": ["taskId", "outputName"], "properties": { "taskId": {"type": "string"}, - "outputName": {"type": "string"} + "outputName": {"type": "string"}, + "type": {"$ref": "#/definitions/TypeSpecType"} }, "additionalProperties": false } diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index de089d7e0e1..77bc4536ff2 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -58,10 +58,12 @@ : graphInput: inputName: string + type: TypeSpecType : taskOutput: taskId: string outputName: string + type: TypeSpecType isEnabled: PredicateType k8sContainerOptions: io.k8s.api.core.v1.Container k8sPodOptions: #PodArgoSubset From eecb0f1e88a98bb4d4b40b2bffc5c83ff7fdca52 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:26:40 -0800 Subject: [PATCH 14/15] Updated the execution options structure --- .../structures/components.json_schema.json | 29 +++++++++++++++---- .../components.json_schema.outline.yaml | 25 +++++++++------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 9e87ad91b6b..73a96460631 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -334,16 +334,36 @@ "additionalProperties": false }, - "PodArgoSubset": { - "description": "Subset of io.k8s.api.core.v1.Pod structure supported by Argo", + "RetryStrategySpec": { + "description": "Optional configuration that specifies how the task should be retried if it fails.", + "type": "object", + "properties": { + "maxRetries": {"type": "integer"} + }, + "additionalProperties": false + }, + + "KubernetesExecutionOptionsSpec": { + "description": "When running on Kubernetes, KubernetesExecutionOptionsSpec describes changes to the configuration of a Kubernetes Pod that will execute the task.", "type": "object", "properties": { "metadata": {"$ref": "#/definitions/ObjectMetaArgoSubset"}, - "spec": {"$ref": "#/definitions/PodSpecArgoSubset"} + "mainContainer": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, + "podSpec": {"$ref": "#/definitions/PodSpecArgoSubset"} }, "additionalProperties": false }, + "ExecutionOptionsSpec": { + "description": "Optional configuration that specifies how the task should be executed. Can be used to set some platform-specific options.", + "type": "object", + "properties": { + "retryStrategy": {"$ref": "#/definitions/RetryStrategySpec"}, + "kubernetesOptions": {"$ref": "#/definitions/KubernetesExecutionOptionsSpec"} + }, + "additionalProperties": false + }, + "TaskSpec": { "description": "'Task specification. Task is a configured component - a component supplied with arguments and other applied configuration changes.", "type": "object", @@ -352,8 +372,7 @@ "componentRef": {"$ref": "#/definitions/ComponentReference"}, "arguments": {"type": "object", "additionalProperties": {"$ref": "#/definitions/ArgumentType"}}, "isEnabled": {"$ref": "#/definitions/PredicateType"}, - "k8sContainerOptions": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, - "k8sPodOptions": {"$ref": "#/definitions/PodArgoSubset"} + "executionOptions": {"$ref": "#/definitions/ExecutionOptionsSpec"} }, "additionalProperties": false }, diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index 77bc4536ff2..a1436077089 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -65,14 +65,17 @@ outputName: string type: TypeSpecType isEnabled: PredicateType - k8sContainerOptions: io.k8s.api.core.v1.Container - k8sPodOptions: #PodArgoSubset - metadata: #ObjectMetaArgoSubset - annotations: Map str -> str - labels: Map str -> str - spec: #PodSpecArgoSubset - activeDeadlineSeconds: integer - affinity: io.k8s.api.core.v1.Affinity - tolerations: io.k8s.api.core.v1.Toleration[] - volumes: io.k8s.api.core.v1.Volume[] - nodeSelector: Map str -> str + executionOptions: #ExecutionOptionsSpec + retryStrategy: #RetryStrategySpec + maxRetries: integer + kubernetesOptions: #KubernetesExecutionOptionsSpec + metadata: #ObjectMetaArgoSubset + annotations: Map str -> str + labels: Map str -> str + mainContainer: io.k8s.api.core.v1.Container + podSpec: #PodSpecArgoSubset + activeDeadlineSeconds: integer + affinity: io.k8s.api.core.v1.Affinity + tolerations: io.k8s.api.core.v1.Toleration[] + volumes: io.k8s.api.core.v1.Volume[] + nodeSelector: Map str -> str From 9b06a542e8714938c76257ccc73aa18f4e09f453 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Tue, 12 Nov 2019 13:34:30 -0800 Subject: [PATCH 15/15] Using the official Kubernetes PodSpec schema instead of Argo's subset --- .../structures/components.json_schema.json | 29 ++----------------- .../components.json_schema.outline.yaml | 11 ++----- 2 files changed, 5 insertions(+), 35 deletions(-) diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/components/structures/components.json_schema.json index 73a96460631..0a7d0b78cc3 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.json +++ b/sdk/python/kfp/components/structures/components.json_schema.json @@ -311,29 +311,6 @@ ] }, - "ObjectMetaArgoSubset": { - "description": "Subset of io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta structure supported by Argo", - "type": "object", - "properties": { - "annotations": {"type": "object", "additionalItems": {"type": "string"}}, - "labels": {"type": "object", "additionalItems": {"type": "string"}} - }, - "additionalProperties": false - }, - - "PodSpecArgoSubset": { - "description": "Subset of Kubernetes PodSpec structure supported by Argo", - "type": "object", - "properties": { - "activeDeadlineSeconds": {"type": "integer"}, - "affinity": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity"}, - "tolerations": {"type": "array", "items": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration"}}, - "volumes": {"type": "array", "items": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Volume"}}, - "nodeSelector": {"type": "object", "additionalItems": {"type": "string"}} - }, - "additionalProperties": false - }, - "RetryStrategySpec": { "description": "Optional configuration that specifies how the task should be retried if it fails.", "type": "object", @@ -347,9 +324,9 @@ "description": "When running on Kubernetes, KubernetesExecutionOptionsSpec describes changes to the configuration of a Kubernetes Pod that will execute the task.", "type": "object", "properties": { - "metadata": {"$ref": "#/definitions/ObjectMetaArgoSubset"}, - "mainContainer": {"$ref": "https://raw.githubusercontent.com/garethr/kubernetes-json-schema/master/v1.11.2/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, - "podSpec": {"$ref": "#/definitions/PodSpecArgoSubset"} + "metadata": {"$ref": "#https://kubernetesjsonschema.dev/v1.14.0/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"}, + "mainContainer": {"$ref": "https://kubernetesjsonschema.dev/v1.14.0/_definitions.json#/definitions/io.k8s.api.core.v1.Container"}, + "podSpec": {"$ref": "https://kubernetesjsonschema.dev/v1.14.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec"} }, "additionalProperties": false }, diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml index a1436077089..f5f92eb08f2 100644 --- a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml +++ b/sdk/python/kfp/components/structures/components.json_schema.outline.yaml @@ -69,13 +69,6 @@ retryStrategy: #RetryStrategySpec maxRetries: integer kubernetesOptions: #KubernetesExecutionOptionsSpec - metadata: #ObjectMetaArgoSubset - annotations: Map str -> str - labels: Map str -> str + metadata: io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta mainContainer: io.k8s.api.core.v1.Container - podSpec: #PodSpecArgoSubset - activeDeadlineSeconds: integer - affinity: io.k8s.api.core.v1.Affinity - tolerations: io.k8s.api.core.v1.Toleration[] - volumes: io.k8s.api.core.v1.Volume[] - nodeSelector: Map str -> str + podSpec: io.k8s.api.core.v1.PodSpec