diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/dsl/_pipeline_param.py index 9ed513465ad..d0ab1afe225 100644 --- a/sdk/python/kfp/dsl/_pipeline_param.py +++ b/sdk/python/kfp/dsl/_pipeline_param.py @@ -141,3 +141,8 @@ def __ge__(self, other): def __hash__(self): return hash((self.op_name, self.name)) + def ignore_type(self): + """ignore_type ignores the type information such that type checking would also pass""" + self.param_type = TypeMeta() + return self + diff --git a/sdk/python/kfp/dsl/_types.py b/sdk/python/kfp/dsl/_types.py index 6ec8a46263e..c142cf35742 100644 --- a/sdk/python/kfp/dsl/_types.py +++ b/sdk/python/kfp/dsl/_types.py @@ -55,15 +55,6 @@ class GCSPath(BaseType): "pattern": "^gs://.*$" } - def __init__(self, path_type='', file_type=''): - ''' - Args - :param path_type: describes the paths, for example, bucket, directory, file, etc - :param file_type: describes the files, for example, JSON, CSV, etc. - ''' - self.path_type = path_type - self.file_type = file_type - class GCRPath(BaseType): openapi_schema_validator = { "type": "string", @@ -121,8 +112,9 @@ def _check_valid_type_dict(payload): if not isinstance(payload[type_name], dict): return False property_types = (int, str, float, bool) + property_value_types = (int, str, float, bool, dict) for property_name in payload[type_name]: - if not isinstance(property_name, property_types) or not isinstance(payload[type_name][property_name], property_types): + if not isinstance(property_name, property_types) or not isinstance(payload[type_name][property_name], property_value_types): return False return True diff --git a/sdk/python/tests/components/test_components.py b/sdk/python/tests/components/test_components.py index 1d133d131ec..a3a915a2ebc 100644 --- a/sdk/python/tests/components/test_components.py +++ b/sdk/python/tests/components/test_components.py @@ -511,7 +511,7 @@ def test_type_check_all_with_types(self): inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcsUri} implementation: @@ -532,7 +532,7 @@ def test_type_check_all_with_types(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -553,14 +553,14 @@ def test_type_check_all_with_types(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_lacking_types(self): + def test_type_check_with_lacking_types(self): component_a = '''\ name: component a description: component a desc inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n} - {name: field_o, type: GcsUri} implementation: @@ -581,7 +581,7 @@ def test_type_check_all_with_lacking_types(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -602,14 +602,14 @@ def test_type_check_all_with_lacking_types(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_inconsistent_types_property_value(self): + def test_type_check_with_inconsistent_types_property_value(self): component_a = '''\ name: component a description: component a desc inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: tsv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: tsv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcsUri} implementation: @@ -630,7 +630,7 @@ def test_type_check_all_with_inconsistent_types_property_value(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -652,14 +652,14 @@ def test_type_check_all_with_inconsistent_types_property_value(self): with self.assertRaises(InconsistentTypeException): b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_inconsistent_types_type_name(self): + def test_type_check_with_inconsistent_types_type_name(self): component_a = '''\ name: component a description: component a desc inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -680,7 +680,7 @@ def test_type_check_all_with_inconsistent_types_type_name(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -702,14 +702,14 @@ def test_type_check_all_with_inconsistent_types_type_name(self): with self.assertRaises(InconsistentTypeException): b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) - def test_type_check_all_with_consistent_types_nonnamed_inputs(self): + def test_type_check_with_consistent_types_nonnamed_inputs(self): component_a = '''\ name: component a description: component a desc inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcsUri} implementation: @@ -730,7 +730,7 @@ def test_type_check_all_with_consistent_types_nonnamed_inputs(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -751,14 +751,14 @@ def test_type_check_all_with_consistent_types_nonnamed_inputs(self): a = task_factory_a(field_l=12) b = task_factory_b(a.outputs['field_n'], field_z=a.outputs['field_m'], field_y=a.outputs['field_o']) - def test_type_check_all_with_inconsistent_types_disabled(self): + def test_type_check_with_inconsistent_types_disabled(self): component_a = '''\ name: component a description: component a desc inputs: - {name: field_l, type: Integer} outputs: - - {name: field_m, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_m, type: {ArtifactA: {path_type: file, file_type: csv}}} - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_o, type: GcrUri} implementation: @@ -779,7 +779,7 @@ def test_type_check_all_with_inconsistent_types_disabled(self): inputs: - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} - {name: field_y, type: GcsUri} - - {name: field_z, type: {GCSPath: {path_type: file, file_type: csv}}} + - {name: field_z, type: {ArtifactA: {path_type: file, file_type: csv}}} outputs: - {name: output_model_uri, type: GcsUri} implementation: @@ -800,5 +800,105 @@ def test_type_check_all_with_inconsistent_types_disabled(self): a = task_factory_a(field_l=12) b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + def test_type_check_with_openapi_shema(self): + component_a = '''\ +name: component a +description: component a desc +inputs: + - {name: field_l, type: Integer} +outputs: + - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gs://.*$ } }}} + - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_o, type: GcrUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3, /pipelines/component/src/train.py] + args: [ + --field-l, {inputValue: field_l}, + ] + fileOutputs: + field_m: /schema.txt + field_n: /feature.txt + field_o: /output.txt +''' + component_b = '''\ +name: component b +description: component b desc +inputs: + - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_y, type: GcrUri} + - {name: field_z, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gs://.*$ } }}} +outputs: + - {name: output_model_uri, type: GcsUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3] + args: [ + --field-x, {inputValue: field_x}, + --field-y, {inputValue: field_y}, + --field-z, {inputValue: field_z}, + ] + fileOutputs: + output_model_uri: /schema.txt +''' + kfp.TYPE_CHECK = True + task_factory_a = comp.load_component_from_text(text=component_a) + task_factory_b = comp.load_component_from_text(text=component_b) + a = task_factory_a(field_l=12) + b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + + def test_type_check_ignore_type(self): + component_a = '''\ +name: component a +description: component a desc +inputs: + - {name: field_l, type: Integer} +outputs: + - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gs://.*$ } }}} + - {name: field_n, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_o, type: GcrUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3, /pipelines/component/src/train.py] + args: [ + --field-l, {inputValue: field_l}, + ] + fileOutputs: + field_m: /schema.txt + field_n: /feature.txt + field_o: /output.txt +''' + component_b = '''\ +name: component b +description: component b desc +inputs: + - {name: field_x, type: {customized_type: {property_a: value_a, property_b: value_b}}} + - {name: field_y, type: GcrUri} + - {name: field_z, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ^gcs://.*$ } }}} +outputs: + - {name: output_model_uri, type: GcsUri} +implementation: + container: + image: gcr.io/ml-pipeline/component-a + command: [python3] + args: [ + --field-x, {inputValue: field_x}, + --field-y, {inputValue: field_y}, + --field-z, {inputValue: field_z}, + ] + fileOutputs: + output_model_uri: /schema.txt +''' + kfp.TYPE_CHECK = True + task_factory_a = comp.load_component_from_text(text=component_a) + task_factory_b = comp.load_component_from_text(text=component_b) + a = task_factory_a(field_l=12) + with self.assertRaises(InconsistentTypeException): + b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'].ignore_type()) + if __name__ == '__main__': unittest.main() diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index a6c596fd7bd..b6b9d244a5a 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -29,15 +29,15 @@ def _set_metadata(self, component_meta): self._metadata = component_meta @component - def componentA(a: {'Schema': {'file_type': 'csv'}}, b: Integer() = 12, c: GCSPath(path_type='file', file_type='tsv') = 'gs://hello/world') -> {'model': Integer()}: + def componentA(a: {'ArtifactA': {'file_type': 'csv'}}, b: Integer() = 12, c: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}} = 'gs://hello/world') -> {'model': Integer()}: return MockContainerOp() containerOp = componentA(1,2,c=3) golden_meta = ComponentMeta(name='componentA', description='') - golden_meta.inputs.append(ParameterMeta(name='a', description='', param_type=TypeMeta(name='Schema', properties={'file_type': 'csv'}))) + golden_meta.inputs.append(ParameterMeta(name='a', description='', param_type=TypeMeta(name='ArtifactA', properties={'file_type': 'csv'}))) golden_meta.inputs.append(ParameterMeta(name='b', description='', param_type=TypeMeta(name='Integer'), default=12)) - golden_meta.inputs.append(ParameterMeta(name='c', description='', param_type=TypeMeta(name='GCSPath', properties={'path_type':'file', 'file_type': 'tsv'}), default='gs://hello/world')) + golden_meta.inputs.append(ParameterMeta(name='c', description='', param_type=TypeMeta(name='ArtifactB', properties={'path_type':'file', 'file_type': 'tsv'}), default='gs://hello/world')) golden_meta.outputs.append(ParameterMeta(name='model', description='', param_type=TypeMeta(name='Integer'))) self.assertEqual(containerOp._metadata, golden_meta) @@ -46,7 +46,7 @@ def test_type_check_with_same_representation(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': GCSPath(path_type='file', file_type='tsv'), 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'GcsUri'}: + def a_op(field_l: Integer()) -> {'field_m': GCSPath(), 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'GcsUri'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-a', @@ -63,7 +63,7 @@ def a_op(field_l: Integer()) -> {'field_m': GCSPath(path_type='file', file_type= @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: 'GcsUri', - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-b', @@ -88,7 +88,7 @@ def test_type_check_with_different_represenation(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -105,7 +105,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -130,7 +130,7 @@ def test_type_check_with_inconsistent_types_property_value(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -147,7 +147,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='csv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'csv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -173,7 +173,7 @@ def test_type_check_with_inconsistent_types_type_name(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -190,7 +190,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -216,7 +216,7 @@ def test_type_check_without_types(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -233,7 +233,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -258,7 +258,7 @@ def test_type_check_nonnamed_inputs(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = True @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -275,7 +275,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -300,7 +300,7 @@ def test_type_check_with_inconsistent_types_disabled(self): """Test type check at the decorator.""" kfp.TYPE_CHECK = False @component - def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: + def a_op(field_l: Integer()) -> {'field_m': {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}, 'field_n': {'customized_type': {'property_a': 'value_a', 'property_b': 'value_b'}}, 'field_o': 'Integer'}: return ContainerOp( name = 'operator a', image = 'gcr.io/ml-pipeline/component-b', @@ -317,7 +317,7 @@ def a_op(field_l: Integer()) -> {'field_m': {'GCSPath': {'path_type': 'file', 'f @component def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': 'value_b'}}, field_y: Integer(), - field_z: GCSPath(path_type='file', file_type='tsv')) -> {'output_model_uri': 'GcsUri'}: + field_z: {'ArtifactB': {'path_type': 'file', 'file_type':'tsv'}}) -> {'output_model_uri': 'GcsUri'}: return ContainerOp( name = 'operator b', image = 'gcr.io/ml-pipeline/component-a', @@ -336,4 +336,90 @@ def b_op(field_x: {'customized_type_a': {'property_a': 'value_a', 'property_b': with Pipeline('pipeline') as p: a = a_op(field_l=12) - b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) \ No newline at end of file + b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + + def test_type_check_with_openapi_schema(self): + """Test type check at the decorator.""" + kfp.TYPE_CHECK = True + @component + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: + return ContainerOp( + name = 'operator a', + image = 'gcr.io/ml-pipeline/component-b', + arguments = [ + '--field-l', field_l, + ], + file_outputs = { + 'field_m': '/schema.txt', + 'field_n': '/feature.txt', + 'field_o': '/output.txt' + } + ) + + @component + def b_op(field_x: {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gs://.*$"}'}}, + field_y: Integer(), + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: + return ContainerOp( + name = 'operator b', + image = 'gcr.io/ml-pipeline/component-a', + command = [ + 'python3', + field_x, + ], + arguments = [ + '--field-y', field_y, + '--field-z', field_z, + ], + file_outputs = { + 'output_model_uri': '/schema.txt', + } + ) + + with Pipeline('pipeline') as p: + a = a_op(field_l=12) + b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + + def test_type_check_with_ignore_type(self): + """Test type check at the decorator.""" + kfp.TYPE_CHECK = True + @component + def a_op(field_l: Integer()) -> {'field_m': 'GCSPath', 'field_n': {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gs://.*$"}'}}, 'field_o': 'Integer'}: + return ContainerOp( + name = 'operator a', + image = 'gcr.io/ml-pipeline/component-b', + arguments = [ + '--field-l', field_l, + ], + file_outputs = { + 'field_m': '/schema.txt', + 'field_n': '/feature.txt', + 'field_o': '/output.txt' + } + ) + + @component + def b_op(field_x: {'customized_type': {'openapi_schema_validator': '{"type": "string", "pattern": "^gcs://.*$"}'}}, + field_y: Integer(), + field_z: GCSPath()) -> {'output_model_uri': 'GcsUri'}: + return ContainerOp( + name = 'operator b', + image = 'gcr.io/ml-pipeline/component-a', + command = [ + 'python3', + field_x, + ], + arguments = [ + '--field-y', field_y, + '--field-z', field_z, + ], + file_outputs = { + 'output_model_uri': '/schema.txt', + } + ) + + with Pipeline('pipeline') as p: + a = a_op(field_l=12) + with self.assertRaises(InconsistentTypeException): + b = b_op(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) + b = b_op(field_x=a.outputs['field_n'].ignore_type(), field_y=a.outputs['field_o'], field_z=a.outputs['field_m']) \ No newline at end of file diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index 9f6e6b37852..5a90c2a6eb7 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -20,21 +20,20 @@ class TestTypes(unittest.TestCase): def test_class_to_dict(self): """Test _class_to_dict function.""" - gcspath_dict = _instance_to_dict(GCSPath(path_type='file', file_type='csv')) + gcspath_dict = _instance_to_dict(GCSPath()) golden_dict = { 'GCSPath': { - 'path_type': 'file', - 'file_type': 'csv', + } } self.assertEqual(golden_dict, gcspath_dict) def test_check_types(self): #Core types - typeA = GCSPath(path_type='file', file_type='csv') - typeB = GCSPath(path_type='file', file_type='csv') + typeA = {'ArtifactA': {'path_type': 'file', 'file_type':'csv'}} + typeB = {'ArtifactA': {'path_type': 'file', 'file_type':'csv'}} self.assertTrue(check_types(typeA, typeB)) - typeC = GCSPath(path_type='file', file_type='tsv') + typeC = {'ArtifactA': {'path_type': 'file', 'file_type':'tsv'}} self.assertFalse(check_types(typeA, typeC)) # Custom types