From 7a1eddd893fb181c243c58bd37929af3d8a6b4b6 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Thu, 14 Mar 2019 18:21:56 -0700 Subject: [PATCH 01/12] fix bug: op_to_template resolve the raw arguments by mapping to the argument_inputs but the argument_inputs lost the type information --- sdk/python/kfp/compiler/compiler.py | 10 ++++++---- sdk/python/kfp/dsl/_metadata.py | 3 +++ sdk/python/kfp/dsl/_pipeline_param.py | 8 +++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index d59ead43c98..21a9401c9e7 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -223,9 +223,12 @@ def _process_args(self, raw_args, argument_inputs): matches += _match_serialized_pipelineparam(str(processed_args[i])) unsanitized_argument_inputs = {} for x in list(set(matches)): - sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2])) - unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2])) - + if len(x) == 3 or (len(x) == 4 and x[3] == ''): + sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2])) + unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2])) + elif len(x) == 4: + sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2], TypeMeta.from_dict_or_str(x[3]))) + unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2], TypeMeta.from_dict_or_str(x[3]))) if argument_inputs: for param in argument_inputs: if str(param) in unsanitized_argument_inputs: @@ -257,7 +260,6 @@ def _build_conventional_artifact(name, path): } }, } - processed_arguments = self._process_args(op.arguments, op.argument_inputs) processed_command = self._process_args(op.command, op.argument_inputs) diff --git a/sdk/python/kfp/dsl/_metadata.py b/sdk/python/kfp/dsl/_metadata.py index 92dab99494b..8fc10423528 100644 --- a/sdk/python/kfp/dsl/_metadata.py +++ b/sdk/python/kfp/dsl/_metadata.py @@ -48,6 +48,9 @@ def to_dict_or_str(self): @staticmethod def from_dict_or_str(json): type_meta = TypeMeta() + if isinstance(json, str) and '{' in json: + import ast + json = ast.literal_eval(json) if isinstance(json, dict): if not _check_valid_type_dict(json): raise ValueError(json + ' is not a valid type string') diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 47c62bc5fbf..7ff688d4de3 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -48,7 +48,13 @@ def _extract_pipelineparams(payloads: str or list[str]): matches = [] for payload in payloads: matches += _match_serialized_pipelineparam(payload) - return [PipelineParam(x[1], x[0], x[2]) for x in list(set(matches))] + pipeline_params = [] + for x in list(set(matches)): + if len(x) == 3 or (len(x) == 4 and x[3] == ''): + pipeline_params.append(PipelineParam(x[1], x[0], x[2])) + elif len(x) == 4: + pipeline_params.append(PipelineParam(x[1], x[0], x[2], TypeMeta.from_dict_or_str(x[3]))) + return pipeline_params class PipelineParam(object): """Representing a future value that is passed between pipeline components. From 6eece145a7ec1c4ae16d9c7ca39d2522d0b02da2 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 11:09:31 -0700 Subject: [PATCH 02/12] fix type pattern matching --- sdk/python/kfp/dsl/_metadata.py | 16 ++++++++++++++++ sdk/python/kfp/dsl/_pipeline_param.py | 4 ++-- sdk/python/tests/dsl/pipeline_param_tests.py | 10 ++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/sdk/python/kfp/dsl/_metadata.py b/sdk/python/kfp/dsl/_metadata.py index 8fc10423528..deb8f7583df 100644 --- a/sdk/python/kfp/dsl/_metadata.py +++ b/sdk/python/kfp/dsl/_metadata.py @@ -32,6 +32,21 @@ def serialize(self): def __eq__(self, other): return self.__dict__ == other.__dict__ +def _convert_ordereddict_to_dict_in_str(payload): + ''' _convert_ordereddict_to_dict_in_str converts the ordereddict struct in the serialized string + to dict struct. + TODO: this function assumes only one ordereddict. need to extend for multiple ordereddict.''' + import re + import ast + matches = re.findall(r'OrderedDict\((.+)\)', payload) + for match in matches: + list_dict = ast.literal_eval(match) + real_dict = {} + for item in list_dict: + real_dict[item[0]] = item[1] + payload = re.sub(r'OrderedDict\((.+)\)', str(real_dict), payload) + return payload + class TypeMeta(BaseMeta): def __init__(self, name: str = '', @@ -49,6 +64,7 @@ def to_dict_or_str(self): def from_dict_or_str(json): type_meta = TypeMeta() if isinstance(json, str) and '{' in json: + json = _convert_ordereddict_to_dict_in_str(json) import ast json = ast.literal_eval(json) if isinstance(json, dict): diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 7ff688d4de3..77295cd8732 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -29,7 +29,7 @@ def _match_serialized_pipelineparam(payload: str): Returns: List(tuple())""" - match = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?);type=(.*?)}}', payload) + match = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?);type=(.*?);}}', payload) if len(match) == 0: match = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?)}}', payload) return match @@ -110,7 +110,7 @@ def __str__(self): if self.param_type is None: return '{{pipelineparam:op=%s;name=%s;value=%s}}' % (op_name, self.name, value) else: - return '{{pipelineparam:op=%s;name=%s;value=%s;type=%s}}' % (op_name, self.name, value, self.param_type.serialize()) + return '{{pipelineparam:op=%s;name=%s;value=%s;type=%s;}}' % (op_name, self.name, value, self.param_type.serialize()) def __repr__(self): return str({self.__class__.__name__: self.__dict__}) diff --git a/sdk/python/tests/dsl/pipeline_param_tests.py b/sdk/python/tests/dsl/pipeline_param_tests.py index a50743ea591..7d18147445a 100644 --- a/sdk/python/tests/dsl/pipeline_param_tests.py +++ b/sdk/python/tests/dsl/pipeline_param_tests.py @@ -29,13 +29,13 @@ def test_str_repr(self): """Test string representation.""" p = PipelineParam(name='param1', op_name='op1') - self.assertEqual('{{pipelineparam:op=op1;name=param1;value=;type=}}', str(p)) + self.assertEqual('{{pipelineparam:op=op1;name=param1;value=;type=;}}', str(p)) p = PipelineParam(name='param2') - self.assertEqual('{{pipelineparam:op=;name=param2;value=;type=}}', str(p)) + self.assertEqual('{{pipelineparam:op=;name=param2;value=;type=;}}', str(p)) p = PipelineParam(name='param3', value='value3') - self.assertEqual('{{pipelineparam:op=;name=param3;value=value3;type=}}', str(p)) + self.assertEqual('{{pipelineparam:op=;name=param3;value=value3;type=;}}', str(p)) def test_extract_pipelineparam(self): """Test _extract_pipeleineparam.""" @@ -49,4 +49,6 @@ def test_extract_pipelineparam(self): self.assertListEqual([p1, p2, p3], params) payload = [str(p1) + stuff_chars + str(p2), str(p2) + stuff_chars + str(p3)] params = _extract_pipelineparams(payload) - self.assertListEqual([p1, p2, p3], params) \ No newline at end of file + self.assertListEqual([p1, p2, p3], params) + + #TODO: add more unit tests to cover real type instances \ No newline at end of file From df1bb189f58962e8641d10fd43b9e40ef1678b98 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 12:07:35 -0700 Subject: [PATCH 03/12] convert orderedDict to dict from the component module --- sdk/python/kfp/dsl/_metadata.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/sdk/python/kfp/dsl/_metadata.py b/sdk/python/kfp/dsl/_metadata.py index deb8f7583df..ce4633ccd42 100644 --- a/sdk/python/kfp/dsl/_metadata.py +++ b/sdk/python/kfp/dsl/_metadata.py @@ -32,21 +32,6 @@ def serialize(self): def __eq__(self, other): return self.__dict__ == other.__dict__ -def _convert_ordereddict_to_dict_in_str(payload): - ''' _convert_ordereddict_to_dict_in_str converts the ordereddict struct in the serialized string - to dict struct. - TODO: this function assumes only one ordereddict. need to extend for multiple ordereddict.''' - import re - import ast - matches = re.findall(r'OrderedDict\((.+)\)', payload) - for match in matches: - list_dict = ast.literal_eval(match) - real_dict = {} - for item in list_dict: - real_dict[item[0]] = item[1] - payload = re.sub(r'OrderedDict\((.+)\)', str(real_dict), payload) - return payload - class TypeMeta(BaseMeta): def __init__(self, name: str = '', @@ -64,13 +49,14 @@ def to_dict_or_str(self): def from_dict_or_str(json): type_meta = TypeMeta() if isinstance(json, str) and '{' in json: - json = _convert_ordereddict_to_dict_in_str(json) import ast json = ast.literal_eval(json) if isinstance(json, dict): if not _check_valid_type_dict(json): raise ValueError(json + ' is not a valid type string') type_meta.name, type_meta.properties = list(json.items())[0] + # Convert possible OrderedDict to dict + type_meta.properties = dict(type_meta.properties) elif isinstance(json, str): type_meta.name = json return type_meta From 9e17ce5e87fe39e9d053066a82fa81a5f9c68a91 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 12:18:30 -0700 Subject: [PATCH 04/12] add unit test to the pipelineparam with types --- sdk/python/kfp/dsl/_metadata.py | 1 + sdk/python/tests/dsl/pipeline_param_tests.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sdk/python/kfp/dsl/_metadata.py b/sdk/python/kfp/dsl/_metadata.py index ce4633ccd42..4f9f11acaed 100644 --- a/sdk/python/kfp/dsl/_metadata.py +++ b/sdk/python/kfp/dsl/_metadata.py @@ -48,6 +48,7 @@ def to_dict_or_str(self): @staticmethod def from_dict_or_str(json): type_meta = TypeMeta() + # When it is a serialized string of a dict, convert the string to dict. if isinstance(json, str) and '{' in json: import ast json = ast.literal_eval(json) diff --git a/sdk/python/tests/dsl/pipeline_param_tests.py b/sdk/python/tests/dsl/pipeline_param_tests.py index 7d18147445a..641a09b248d 100644 --- a/sdk/python/tests/dsl/pipeline_param_tests.py +++ b/sdk/python/tests/dsl/pipeline_param_tests.py @@ -15,6 +15,7 @@ from kfp.dsl import PipelineParam from kfp.dsl._pipeline_param import _extract_pipelineparams +from kfp.dsl._metadata import TypeMeta import unittest @@ -51,4 +52,14 @@ def test_extract_pipelineparam(self): params = _extract_pipelineparams(payload) self.assertListEqual([p1, p2, p3], params) - #TODO: add more unit tests to cover real type instances \ No newline at end of file + def test_extract_pipelineparam_with_types(self): + p1 = PipelineParam(name='param1', op_name='op1', param_type=TypeMeta(name='customized_type_a', properties={'property_a': 'value_a'})) + p2 = PipelineParam(name='param2', param_type=TypeMeta(name='customized_type_b')) + p3 = PipelineParam(name='param3', value='value3', param_type=TypeMeta(name='customized_type_c', properties={'property_c': 'value_c'})) + stuff_chars = ' between ' + payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) + params = _extract_pipelineparams(payload) + self.assertListEqual([p1, p2, p3], params) + payload = [str(p1) + stuff_chars + str(p2), str(p2) + stuff_chars + str(p3)] + params = _extract_pipelineparams(payload) + self.assertListEqual([p1, p2, p3], params) \ No newline at end of file From 91f654ce8450891bd0ae80038c4ce3a462f3a5fb Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 14:03:51 -0700 Subject: [PATCH 05/12] create TypeMeta deserialize function, add comments --- sdk/python/kfp/compiler/compiler.py | 6 ++-- sdk/python/kfp/dsl/_metadata.py | 39 +++++++++++++++++--------- sdk/python/kfp/dsl/_pipeline_param.py | 2 +- sdk/python/tests/dsl/metadata_tests.py | 6 ++-- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index 21a9401c9e7..2f08c0b9304 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -223,12 +223,14 @@ def _process_args(self, raw_args, argument_inputs): matches += _match_serialized_pipelineparam(str(processed_args[i])) unsanitized_argument_inputs = {} for x in list(set(matches)): + # When the serialized pipelineparam does not contain the type information if len(x) == 3 or (len(x) == 4 and x[3] == ''): sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2])) unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2])) + # When the serialized pipelineparam contains the type information elif len(x) == 4: - sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2], TypeMeta.from_dict_or_str(x[3]))) - unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2], TypeMeta.from_dict_or_str(x[3]))) + sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2], TypeMeta.deserialize(x[3]))) + unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2], TypeMeta.deserialize(x[3]))) if argument_inputs: for param in argument_inputs: if str(param) in unsanitized_argument_inputs: diff --git a/sdk/python/kfp/dsl/_metadata.py b/sdk/python/kfp/dsl/_metadata.py index 4f9f11acaed..98426333816 100644 --- a/sdk/python/kfp/dsl/_metadata.py +++ b/sdk/python/kfp/dsl/_metadata.py @@ -46,25 +46,36 @@ def to_dict_or_str(self): return {self.name: self.properties} @staticmethod - def from_dict_or_str(json): + def from_dict_or_str(payload): + '''from_dict_or_str accepts a payload object and returns a TypeMeta instance + Args: + payload (str/dict): the payload could be a str or a dict + ''' + type_meta = TypeMeta() - # When it is a serialized string of a dict, convert the string to dict. - if isinstance(json, str) and '{' in json: - import ast - json = ast.literal_eval(json) - if isinstance(json, dict): - if not _check_valid_type_dict(json): - raise ValueError(json + ' is not a valid type string') - type_meta.name, type_meta.properties = list(json.items())[0] + if isinstance(payload, dict): + if not _check_valid_type_dict(payload): + raise ValueError(payload + ' is not a valid type string') + type_meta.name, type_meta.properties = list(payload.items())[0] # Convert possible OrderedDict to dict type_meta.properties = dict(type_meta.properties) - elif isinstance(json, str): - type_meta.name = json + elif isinstance(payload, str): + type_meta.name = payload return type_meta def serialize(self): return str(self.to_dict_or_str()) + @staticmethod + def deserialize(payload): + # If the payload is a string of a dict serialization, convert it back to a dict + try: + import ast + payload = ast.literal_eval(payload) + except: + pass + return TypeMeta.from_dict_or_str(payload) + class ParameterMeta(BaseMeta): def __init__(self, name: str, @@ -129,13 +140,13 @@ def _annotation_to_typemeta(annotation): TypeMeta ''' if isinstance(annotation, BaseType): - arg_type = TypeMeta.from_dict_or_str(_instance_to_dict(annotation)) + arg_type = TypeMeta.deserialize(_instance_to_dict(annotation)) elif isinstance(annotation, str): - arg_type = TypeMeta.from_dict_or_str(annotation) + arg_type = TypeMeta.deserialize(annotation) elif isinstance(annotation, dict): if not _check_valid_type_dict(annotation): raise ValueError('Annotation ' + str(annotation) + ' is not a valid type dictionary.') - arg_type = TypeMeta.from_dict_or_str(annotation) + arg_type = TypeMeta.deserialize(annotation) else: return TypeMeta() return arg_type diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 77295cd8732..9ced7b0fc85 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -53,7 +53,7 @@ def _extract_pipelineparams(payloads: str or list[str]): if len(x) == 3 or (len(x) == 4 and x[3] == ''): pipeline_params.append(PipelineParam(x[1], x[0], x[2])) elif len(x) == 4: - pipeline_params.append(PipelineParam(x[1], x[0], x[2], TypeMeta.from_dict_or_str(x[3]))) + pipeline_params.append(PipelineParam(x[1], x[0], x[2], TypeMeta.deserialize(x[3]))) return pipeline_params class PipelineParam(object): diff --git a/sdk/python/tests/dsl/metadata_tests.py b/sdk/python/tests/dsl/metadata_tests.py index 24a174ce257..52c5c161471 100644 --- a/sdk/python/tests/dsl/metadata_tests.py +++ b/sdk/python/tests/dsl/metadata_tests.py @@ -16,7 +16,7 @@ import unittest class TestTypeMeta(unittest.TestCase): - def test_from_dict_or_str(self): + def test_deserialize(self): component_dict = { 'GCSPath': { 'bucket_type': 'directory', @@ -25,11 +25,11 @@ def test_from_dict_or_str(self): } golden_type_meta = TypeMeta(name='GCSPath', properties={'bucket_type': 'directory', 'file_type': 'csv'}) - self.assertEqual(TypeMeta.from_dict_or_str(component_dict), golden_type_meta) + self.assertEqual(TypeMeta.deserialize(component_dict), golden_type_meta) component_str = 'GCSPath' golden_type_meta = TypeMeta(name='GCSPath') - self.assertEqual(TypeMeta.from_dict_or_str(component_str), golden_type_meta) + self.assertEqual(TypeMeta.deserialize(component_str), golden_type_meta) def test_eq(self): From 53300a994c9d4abd026477b99c8ecb624ccb0dc1 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 14:33:28 -0700 Subject: [PATCH 06/12] strongly typed pipelineparamtuple --- sdk/python/kfp/compiler/compiler.py | 16 +++++---------- sdk/python/kfp/dsl/_pipeline_param.py | 29 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index 2f08c0b9304..f5155194837 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -219,18 +219,12 @@ def _process_args(self, raw_args, argument_inputs): processed_args = list(map(str, raw_args)) for i, _ in enumerate(processed_args): # unsanitized_argument_inputs stores a dict: string of sanitized param -> string of unsanitized param - matches = [] - matches += _match_serialized_pipelineparam(str(processed_args[i])) + param_tuples = [] + param_tuples += _match_serialized_pipelineparam(str(processed_args[i])) unsanitized_argument_inputs = {} - for x in list(set(matches)): - # When the serialized pipelineparam does not contain the type information - if len(x) == 3 or (len(x) == 4 and x[3] == ''): - sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2])) - unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2])) - # When the serialized pipelineparam contains the type information - elif len(x) == 4: - sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(x[1]), K8sHelper.sanitize_k8s_name(x[0]), x[2], TypeMeta.deserialize(x[3]))) - unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(x[1], x[0], x[2], TypeMeta.deserialize(x[3]))) + for param_tuple in list(set(param_tuples)): + sanitized_str = str(dsl.PipelineParam(K8sHelper.sanitize_k8s_name(param_tuple.name), K8sHelper.sanitize_k8s_name(param_tuple.op), param_tuple.value, TypeMeta.deserialize(param_tuple.type))) + unsanitized_argument_inputs[sanitized_str] = str(dsl.PipelineParam(param_tuple.name, param_tuple.op, param_tuple.value, TypeMeta.deserialize(param_tuple.type))) if argument_inputs: for param in argument_inputs: if str(param) in unsanitized_argument_inputs: diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 9ced7b0fc85..9ed513465ad 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -21,6 +21,7 @@ # TODO: Move this to a separate class # For now, this identifies a condition with only "==" operator supported. ConditionOperator = namedtuple('ConditionOperator', 'operator operand1 operand2') +PipelineParamTuple = namedtuple('PipelineParamTuple', 'name op value type') def _match_serialized_pipelineparam(payload: str): """_match_serialized_pipelineparam matches the serialized pipelineparam. @@ -28,11 +29,18 @@ def _match_serialized_pipelineparam(payload: str): payloads (str): a string that contains the serialized pipelineparam. Returns: - List(tuple())""" - match = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?);type=(.*?);}}', payload) - if len(match) == 0: - match = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?)}}', payload) - return match + PipelineParamTuple + """ + matches = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?);type=(.*?);}}', payload) + if len(matches) == 0: + matches = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+);value=(.*?)}}', payload) + param_tuples = [] + for match in matches: + if len(match) == 3: + param_tuples.append(PipelineParamTuple(name=match[1], op=match[0], value=match[2], type='')) + elif len(match) == 4: + param_tuples.append(PipelineParamTuple(name=match[1], op=match[0], value=match[2], type=match[3])) + return param_tuples def _extract_pipelineparams(payloads: str or list[str]): """_extract_pipelineparam extract a list of PipelineParam instances from the payload string. @@ -45,15 +53,12 @@ def _extract_pipelineparams(payloads: str or list[str]): """ if isinstance(payloads, str): payloads = [payloads] - matches = [] + param_tuples = [] for payload in payloads: - matches += _match_serialized_pipelineparam(payload) + param_tuples += _match_serialized_pipelineparam(payload) pipeline_params = [] - for x in list(set(matches)): - if len(x) == 3 or (len(x) == 4 and x[3] == ''): - pipeline_params.append(PipelineParam(x[1], x[0], x[2])) - elif len(x) == 4: - pipeline_params.append(PipelineParam(x[1], x[0], x[2], TypeMeta.deserialize(x[3]))) + for param_tuple in list(set(param_tuples)): + pipeline_params.append(PipelineParam(param_tuple.name, param_tuple.op, param_tuple.value, TypeMeta.deserialize(param_tuple.type))) return pipeline_params class PipelineParam(object): From 17922d178ee3e8a036869b850255cc26de8636ad Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 14:44:57 -0700 Subject: [PATCH 07/12] remove GCSPath fields to avoid artifact type confusion change the type json schema field name to openAPIV3Schema --- sdk/python/kfp/dsl/_types.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index 6ec8a46263e..788d836993f 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -19,72 +19,63 @@ class BaseType: # Primitive Types class Integer(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "integer" } class String(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "string" } class Float(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "number" } class Bool(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "boolean" } class List(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "array" } class Dict(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "object", } # GCP Types class GCSPath(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "string", "pattern": "^gs://.*$" } - def __init__(self, path_type='', file_type=''): - ''' - Args - :param path_type: describes the paths, for example, bucket, directory, file, etc - :param file_type: describes the files, for example, JSON, CSV, etc. - ''' - self.path_type = path_type - self.file_type = file_type - class GCRPath(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "string", "pattern": "^.*gcr\\.io/.*$" } class GCPRegion(BaseType): - openapi_schema_validator = { + openAPIV3Schema = { "type": "string" } class GCPProjectID(BaseType): '''MetaGCPProjectID: GCP project id''' - openapi_schema_validator = { + openAPIV3Schema = { "type": "string" } # General Types class LocalPath(BaseType): #TODO: add restriction to path - openapi_schema_validator = { + openAPIV3Schema = { "type": "string" } From efa925086d9856e1875ade94619df573c2fea703 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 15:07:52 -0700 Subject: [PATCH 08/12] fix unit tests; add unit test for openapishema property --- sdk/python/tests/dsl/component_tests.py | 62 +++++++++++++++++++++---- sdk/python/tests/dsl/type_tests.py | 11 ++--- 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index a6c596fd7bd..e2d4883b07c 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -29,7 +29,7 @@ def _set_metadata(self, component_meta): self._metadata = component_meta @component - def componentA(a: {'Schema': {'file_type': 'csv'}}, b: Integer() = 12, c: GCSPath(path_type='file', file_type='tsv') = 'gs://hello/world') -> {'model': Integer()}: + def componentA(a: {'Schema': {'file_type': 'csv'}}, b: Integer() = 12, c: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}} = 'gs://hello/world') -> {'model': Integer()}: return MockContainerOp() containerOp = componentA(1,2,c=3) @@ -46,7 +46,7 @@ def test_type_check_with_same_representation(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': GCSPath(path_type='file', file_type='tsv'), 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'GcsUri'}: + def a_op(field_l: Integer()) -> {'field_m': GCSPath(), 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'GcsUri'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-a', @@ -63,7 +63,7 @@ def a_op(field_l: Integer()) -> {'field_m': GCSPath(path_type='file', file_type= @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: 'GcsUri', - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-b', @@ -88,7 +88,7 @@ def test_type_check_with_different_represenation(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -105,7 +105,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -147,7 +147,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='csv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'GCSPath': {'path_type': 'file', 'file_type':'csv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -190,7 +190,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -233,7 +233,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -275,7 +275,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -317,7 +317,49 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: + return ContainerOp( + name = 'operator b', + image = 'gcr.io/ml-pipeline/component-a', + command = [ + 'python3', + field_x, + ], + arguments = [ + '--field-y', field_y, + '--field-z', field_z, + ], + file_outputs = { + 'output_model_uri': '/schema.txt', + } + ) + + with Pipeline('pipeline') as p: + a = a_op(field_l=12) + b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + + def test_type_check_with_openapi_schema(self): + """Test type check at the decorator.""" + kfp.TYPE_CHECK = True + @component + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^.*gcr\\.io/.*$"}'}}, 'field_o': 'Integer'}: + return ContainerOp( + name = 'operator a', + image = 'gcr.io/ml-pipeline/component-b', + arguments = [ + '--field-l', field_l, + ], + file_outputs = { + 'field_m': '/schema.txt', + 'field_n': '/feature.txt', + 'field_o': '/output.txt' + } + ) + + @component + def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^.*gcr\\.io/.*$"}'}}, + field_y: Integer(), + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index 9f6e6b37852..332d2bad4e0 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -20,21 +20,20 @@ class TestTypes(unittest.TestCase): def test_class_to_dict(self): """Test _class_to_dict function.""" - gcspath_dict = _instance_to_dict(GCSPath(path_type='file', file_type='csv')) + gcspath_dict = _instance_to_dict(GCSPath()) golden_dict = { 'GCSPath': { - 'path_type': 'file', - 'file_type': 'csv', + } } self.assertEqual(golden_dict, gcspath_dict) def test_check_types(self): #Core types - typeA = GCSPath(path_type='file', file_type='csv') - typeB = GCSPath(path_type='file', file_type='csv') + typeA = {'GCSPath': {'path_type': 'file', 'file_type':'csv'}} + typeB = {'GCSPath': {'path_type': 'file', 'file_type':'csv'}} self.assertTrue(check_types(typeA, typeB)) - typeC = GCSPath(path_type='file', file_type='tsv') + typeC = {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}} self.assertFalse(check_types(typeA, typeC)) # Custom types From 2ec1169ac5878666f79424534095f418b6c1e3a6 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Fri, 15 Mar 2019 15:25:57 -0700 Subject: [PATCH 09/12] add unit test at the component module; fix bug --- sdk/python/kfp/dsl/_types.py | 3 +- .../tests/components/test_components.py | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index 788d836993f..f9917dcd35f 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -112,8 +112,9 @@ def _check_valid_type_dict(payload): if not isinstance(payload[type_name], dict): return False property_types = (int, str, float, bool) + property_value_types = (int, str, float, bool, dict) for property_name in payload[type_name]: - if not isinstance(property_name, property_types) or not isinstance(payload[type_name][property_name], property_types): + if not isinstance(property_name, property_types) or not isinstance(payload[type_name][property_name], property_value_types): return False return True diff --git a/sdk/python/tests/components/test_components.py b/sdk/python/tests/components/test_components.py index 1d133d131ec..06a1a610223 100644 --- a/sdk/python/tests/components/test_components.py +++ b/sdk/python/tests/components/test_components.py @@ -800,5 +800,54 @@ def test_type_check_all_with_inconsistent_types_disabled(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + def test_type_check_all_with_openapi_shema(self): + component_a = '''\ +name: component a +description: component a desc +inputs: + - {name: field_l, type: Integer} +outputs: + - {name: field_m, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^.*gcr\\.io/.*$ } }}} + - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_o, type: GcrUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3, /pipelines/component/src/train.py] + args: [ + --field-l, {inputValue: field_l}, + ] + fileOutputs: + field_m: /schema.txt + field_n: /feature.txt + field_o: /output.txt +''' + component_b = '''\ +name: component b +description: component b desc +inputs: + - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_y, type: GcrUri} + - {name: field_z, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^.*gcr\\.io/.*$} }}} +outputs: + - {name: output_model_uri, type: GcsUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3] + args: [ + --field-x, {inputValue: field_x}, + --field-y, {inputValue: field_y}, + --field-z, {inputValue: field_z}, + ] + fileOutputs: + output_model_uri: /schema.txt +''' + kfp.TYPE_CHECK = True + task_factory_a = comp.load_component_from_text(text=component_a) + task_factory_b = comp.load_component_from_text(text=component_b) + a = task_factory_a(field_l=12) + b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + if __name__ == '__main__': unittest.main() From db947b1a090a497cc1d193402d3d3c2c316a3d77 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Sun, 17 Mar 2019 09:29:09 -0700 Subject: [PATCH 10/12] add ignore_type in pipelineparam --- sdk/python/kfp/dsl/_pipeline_param.py | 5 ++ .../tests/components/test_components.py | 67 ++++++++++++++++--- sdk/python/tests/dsl/component_tests.py | 50 +++++++++++++- 3 files changed, 111 insertions(+), 11 deletions(-) diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 9ed513465ad..d0ab1afe225 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -141,3 +141,8 @@ def __ge__(self, other): def __hash__(self): return hash((self.op_name, self.name)) + def ignore_type(self): + """ignore_type ignores the type information such that type checking would also pass""" + self.param_type = TypeMeta() + return self + diff --git a/sdk/python/tests/components/test_components.py b/sdk/python/tests/components/test_components.py index 06a1a610223..6c64055017b 100644 --- a/sdk/python/tests/components/test_components.py +++ b/sdk/python/tests/components/test_components.py @@ -553,7 +553,7 @@ def test_type_check_all_with_types(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_lacking_types(self): + def test_type_check_with_lacking_types(self): component_a = '''\ name: component a description: component a desc @@ -602,7 +602,7 @@ def test_type_check_all_with_lacking_types(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_inconsistent_types_property_value(self): + def test_type_check_with_inconsistent_types_property_value(self): component_a = '''\ name: component a description: component a desc @@ -652,7 +652,7 @@ def test_type_check_all_with_inconsistent_types_property_value(self): with self.assertRaises(InconsistentTypeException): b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_inconsistent_types_type_name(self): + def test_type_check_with_inconsistent_types_type_name(self): component_a = '''\ name: component a description: component a desc @@ -702,7 +702,7 @@ def test_type_check_all_with_inconsistent_types_type_name(self): with self.assertRaises(InconsistentTypeException): b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_consistent_types_nonnamed_inputs(self): + def test_type_check_with_consistent_types_nonnamed_inputs(self): component_a = '''\ name: component a description: component a desc @@ -751,7 +751,7 @@ def test_type_check_all_with_consistent_types_nonnamed_inputs(self): a = task_factory_a(field_l=12) b = task_factory_b(a.outputs['field_n'], field_z=a.outputs['field_m'], field_y=a.outputs['field_o']) - def test_type_check_all_with_inconsistent_types_disabled(self): + def test_type_check_with_inconsistent_types_disabled(self): component_a = '''\ name: component a description: component a desc @@ -800,14 +800,14 @@ def test_type_check_all_with_inconsistent_types_disabled(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_openapi_shema(self): + def test_type_check_with_openapi_shema(self): component_a = '''\ name: component a description: component a desc inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^.*gcr\\.io/.*$ } }}} + - {name: field_m, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gs://.*$ } }}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -828,7 +828,7 @@ def test_type_check_all_with_openapi_shema(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcrUri} - - {name: field_z, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^.*gcr\\.io/.*$} }}} + - {name: field_z, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gs://.*$ } }}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -849,5 +849,56 @@ def test_type_check_all_with_openapi_shema(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + def test_type_check_ignore_type(self): + component_a = '''\ +name: component a +description: component a desc +inputs: + - {name: field_l, type: Integer} +outputs: + - {name: field_m, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gs://.*$ } }}} + - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_o, type: GcrUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3, /pipelines/component/src/train.py] + args: [ + --field-l, {inputValue: field_l}, + ] + fileOutputs: + field_m: /schema.txt + field_n: /feature.txt + field_o: /output.txt +''' + component_b = '''\ +name: component b +description: component b desc +inputs: + - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_y, type: GcrUri} + - {name: field_z, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gcs://.*$ } }}} +outputs: + - {name: output_model_uri, type: GcsUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3] + args: [ + --field-x, {inputValue: field_x}, + --field-y, {inputValue: field_y}, + --field-z, {inputValue: field_z}, + ] + fileOutputs: + output_model_uri: /schema.txt +''' + kfp.TYPE_CHECK = True + task_factory_a = comp.load_component_from_text(text=component_a) + task_factory_b = comp.load_component_from_text(text=component_b) + a = task_factory_a(field_l=12) + with self.assertRaises(InconsistentTypeException): + b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'].ignore_type()) + if __name__ == '__main__': unittest.main() diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index e2d4883b07c..2f0432a68a4 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -342,7 +342,7 @@ def test_type_check_with_openapi_schema(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^.*gcr\\.io/.*$"}'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -357,7 +357,7 @@ def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_t ) @component - def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^.*gcr\\.io/.*$"}'}}, + def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gs://.*$"}'}}, field_y: Integer(), field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( @@ -378,4 +378,48 @@ def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "p with Pipeline('pipeline') as p: a = a_op(field_l=12) - b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) \ No newline at end of file + b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + + def test_type_check_with_ignore_type(self): + """Test type check at the decorator.""" + kfp.TYPE_CHECK = True + @component + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: + return ContainerOp( + name = 'operator a', + image = 'gcr.io/ml-pipeline/component-b', + arguments = [ + '--field-l', field_l, + ], + file_outputs = { + 'field_m': '/schema.txt', + 'field_n': '/feature.txt', + 'field_o': '/output.txt' + } + ) + + @component + def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gcs://.*$"}'}}, + field_y: Integer(), + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: + return ContainerOp( + name = 'operator b', + image = 'gcr.io/ml-pipeline/component-a', + command = [ + 'python3', + field_x, + ], + arguments = [ + '--field-y', field_y, + '--field-z', field_z, + ], + file_outputs = { + 'output_model_uri': '/schema.txt', + } + ) + + with Pipeline('pipeline') as p: + a = a_op(field_l=12) + with self.assertRaises(InconsistentTypeException): + b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + b = b_op(field_x=a.outputs['field_n'].ignore_type(), field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) \ No newline at end of file From 0845dd24257a99de79c31f800a50fc48e4fc044e Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Mon, 18 Mar 2019 18:18:52 -0700 Subject: [PATCH 11/12] change the names in the artifact types to avoid confusion with the parameter types --- .../tests/components/test_components.py | 24 ++++++++--------- sdk/python/tests/dsl/component_tests.py | 26 +++++++++---------- sdk/python/tests/dsl/type_tests.py | 6 ++--- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/sdk/python/tests/components/test_components.py b/sdk/python/tests/components/test_components.py index 6c64055017b..e6ffae702ec 100644 --- a/sdk/python/tests/components/test_components.py +++ b/sdk/python/tests/components/test_components.py @@ -511,7 +511,7 @@ def test_type_check_all_with_types(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcsUri} implementation: @@ -532,7 +532,7 @@ def test_type_check_all_with_types(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -560,7 +560,7 @@ def test_type_check_with_lacking_types(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n} - {name: field_o, type: GcsUri} implementation: @@ -581,7 +581,7 @@ def test_type_check_with_lacking_types(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -609,7 +609,7 @@ def test_type_check_with_inconsistent_types_property_value(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: tsv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: tsv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcsUri} implementation: @@ -630,7 +630,7 @@ def test_type_check_with_inconsistent_types_property_value(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -659,7 +659,7 @@ def test_type_check_with_inconsistent_types_type_name(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -680,7 +680,7 @@ def test_type_check_with_inconsistent_types_type_name(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -709,7 +709,7 @@ def test_type_check_with_consistent_types_nonnamed_inputs(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcsUri} implementation: @@ -730,7 +730,7 @@ def test_type_check_with_consistent_types_nonnamed_inputs(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -758,7 +758,7 @@ def test_type_check_with_inconsistent_types_disabled(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -779,7 +779,7 @@ def test_type_check_with_inconsistent_types_disabled(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index 2f0432a68a4..fb8a4eb798e 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -29,15 +29,15 @@ def _set_metadata(self, component_meta): self._metadata = component_meta @component - def componentA(a: {'Schema': {'file_type': 'csv'}}, b: Integer() = 12, c: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}} = 'gs://hello/world') -> {'model': Integer()}: + def componentA(a: {'ArtifactA': {'file_type': 'csv'}}, b: Integer() = 12, c: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}} = 'gs://hello/world') -> {'model': Integer()}: return MockContainerOp() containerOp = componentA(1,2,c=3) golden_meta = ComponentMeta(name='componentA', description='') - golden_meta.inputs.append(ParameterMeta(name='a', description='', param_type=TypeMeta(name='Schema', properties={'file_type': 'csv'}))) + golden_meta.inputs.append(ParameterMeta(name='a', description='', param_type=TypeMeta(name='ArtifactA', properties={'file_type': 'csv'}))) golden_meta.inputs.append(ParameterMeta(name='b', description='', param_type=TypeMeta(name='Integer'), default=12)) - golden_meta.inputs.append(ParameterMeta(name='c', description='', param_type=TypeMeta(name='GCSPath', properties={'path_type':'file', 'file_type': 'tsv'}), default='gs://hello/world')) + golden_meta.inputs.append(ParameterMeta(name='c', description='', param_type=TypeMeta(name='ArtifactB', properties={'path_type':'file', 'file_type': 'tsv'}), default='gs://hello/world')) golden_meta.outputs.append(ParameterMeta(name='model', description='', param_type=TypeMeta(name='Integer'))) self.assertEqual(containerOp._metadata, golden_meta) @@ -130,7 +130,7 @@ def test_type_check_with_inconsistent_types_property_value(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -147,7 +147,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: {'GCSPath': {'path_type': 'file', 'file_type':'csv'}}) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'csv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -173,7 +173,7 @@ def test_type_check_with_inconsistent_types_type_name(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -190,7 +190,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -216,7 +216,7 @@ def test_type_check_without_types(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -233,7 +233,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x, field_y: Integer(), - field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -258,7 +258,7 @@ def test_type_check_nonnamed_inputs(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -275,7 +275,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x, field_y: Integer(), - field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -300,7 +300,7 @@ def test_type_check_with_inconsistent_types_disabled(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = False @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -317,7 +317,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index 332d2bad4e0..5a90c2a6eb7 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -30,10 +30,10 @@ def test_class_to_dict(self): def test_check_types(self): #Core types - typeA = {'GCSPath': {'path_type': 'file', 'file_type':'csv'}} - typeB = {'GCSPath': {'path_type': 'file', 'file_type':'csv'}} + typeA = {'ArtifactA': {'path_type': 'file', 'file_type':'csv'}} + typeB = {'ArtifactA': {'path_type': 'file', 'file_type':'csv'}} self.assertTrue(check_types(typeA, typeB)) - typeC = {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}} + typeC = {'ArtifactA': {'path_type': 'file', 'file_type':'tsv'}} self.assertFalse(check_types(typeA, typeC)) # Custom types From f0ad48719d4604a30280ce6b0f989631dfe647e4 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Tue, 19 Mar 2019 09:25:53 -0700 Subject: [PATCH 12/12] based on the google python style guide, change the camel case to lower case with underscores --- sdk/python/kfp/dsl/_types.py | 22 +++++++++---------- .../tests/components/test_components.py | 8 +++---- sdk/python/tests/dsl/component_tests.py | 8 +++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index f9917dcd35f..c142cf35742 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -19,63 +19,63 @@ class BaseType: # Primitive Types class Integer(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "integer" } class String(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "string" } class Float(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "number" } class Bool(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "boolean" } class List(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "array" } class Dict(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "object", } # GCP Types class GCSPath(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "string", "pattern": "^gs://.*$" } class GCRPath(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "string", "pattern": "^.*gcr\\.io/.*$" } class GCPRegion(BaseType): - openAPIV3Schema = { + openapi_schema_validator = { "type": "string" } class GCPProjectID(BaseType): '''MetaGCPProjectID: GCP project id''' - openAPIV3Schema = { + openapi_schema_validator = { "type": "string" } # General Types class LocalPath(BaseType): #TODO: add restriction to path - openAPIV3Schema = { + openapi_schema_validator = { "type": "string" } diff --git a/sdk/python/tests/components/test_components.py b/sdk/python/tests/components/test_components.py index e6ffae702ec..a3a915a2ebc 100644 --- a/sdk/python/tests/components/test_components.py +++ b/sdk/python/tests/components/test_components.py @@ -807,7 +807,7 @@ def test_type_check_with_openapi_shema(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gs://.*$ } }}} + - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gs://.*$ } }}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -828,7 +828,7 @@ def test_type_check_with_openapi_shema(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcrUri} - - {name: field_z, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gs://.*$ } }}} + - {name: field_z, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gs://.*$ } }}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -856,7 +856,7 @@ def test_type_check_ignore_type(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gs://.*$ } }}} + - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gs://.*$ } }}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -877,7 +877,7 @@ def test_type_check_ignore_type(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcrUri} - - {name: field_z, type: {GCSPath: {openAPIV3Schema: {type: string, pattern: ^gcs://.*$ } }}} + - {name: field_z, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gcs://.*$ } }}} outputs: - {name: output_model_uri, type: GcsUri} implementation: diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index fb8a4eb798e..b6b9d244a5a 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -342,7 +342,7 @@ def test_type_check_with_openapi_schema(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -357,7 +357,7 @@ def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_t ) @component - def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gs://.*$"}'}}, + def b_op(field_x: {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gs://.*$"}'}}, field_y: Integer(), field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( @@ -384,7 +384,7 @@ def test_type_check_with_ignore_type(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -399,7 +399,7 @@ def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_t ) @component - def b_op(field_x: {'customized_type': {'openAPIV3Schema': '{"type": "string", "pattern": "^gcs://.*$"}'}}, + def b_op(field_x: {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gcs://.*$"}'}}, field_y: Integer(), field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp(