SDK - Persisting all output values

sdk/python/kfp/compiler/_op_to_template.py

@@ -190,6 +190,8 @@ def _op_to_template(op: BaseOp):
     if isinstance(op, dsl.ContainerOp):
         # default output artifacts
         output_artifact_paths = OrderedDict(op.output_artifact_paths)
+        # This should have been as easy as output_artifact_paths.update(op.file_outputs), but the _outputs_to_json function changes the output names and we must do the same here, so that the names are the same
+        output_artifact_paths.update(sorted(((param.full_name, processed_op.file_outputs[param.name]) for param in processed_op.outputs.values()), key=lambda x: x[0]))
 
         output_artifacts = [
              K8sHelper.convert_k8s_obj_to_json(

sdk/python/tests/compiler/compiler_tests.py

@@ -98,6 +98,12 @@ def my_pipeline(msg1, json, kind, msg2='value2'):
           ]},
         'name': 'echo',
         'outputs': {
+          'artifacts': [
+            {
+              'name': 'echo-merged',
+              'path': '/tmp/message.txt',
+            },
+          ],
           'parameters': [
             {'name': 'echo-merged',
             'valueFrom': {'path': '/tmp/message.txt'}
@@ -571,7 +577,8 @@ def _test_op_to_template_yaml(self, ops, file_base_name):
     compiled_template = compiler._op_to_template._op_to_template(ops)
 
     del compiled_template['name'], expected['name']
-    del compiled_template['outputs']['parameters'][0]['name'], expected['outputs']['parameters'][0]['name']
+    for output in compiled_template['outputs'].get('parameters', []) + compiled_template['outputs'].get('artifacts', []) + expected['outputs'].get('parameters', []) + expected['outputs'].get('artifacts', []):
+      del output['name']
     assert compiled_template == expected
 
   def test_tolerations(self):

sdk/python/tests/compiler/testdata/basic.yaml

@@ -73,6 +73,9 @@ spec:
       - name: message
     name: get-frequent
     outputs:
+      artifacts:
+      - name: get-frequent-word
+        path: /tmp/message.txt
       parameters:
       - name: get-frequent-word
         valueFrom:

sdk/python/tests/compiler/testdata/basic_no_decorator.yaml

@@ -74,6 +74,9 @@ spec:
           - name: message
       name: get-frequent
       outputs:
+        artifacts:
+          - name: get-frequent-word
+            path: /tmp/message.txt
         parameters:
           - name: get-frequent-word
             valueFrom:

sdk/python/tests/compiler/testdata/coin.yaml

@@ -91,6 +91,9 @@ spec:
       image: python:alpine3.6
     name: flip
     outputs:
+      artifacts:
+      - name: flip-output
+        path: /tmp/output
       parameters:
       - name: flip-output
         valueFrom:
@@ -105,6 +108,9 @@ spec:
       image: python:alpine3.6
     name: flip-again
     outputs:
+      artifacts:
+      - name: flip-again-output
+        path: /tmp/output
       parameters:
       - name: flip-again-output
         valueFrom:

sdk/python/tests/compiler/testdata/compose.yaml

@@ -37,6 +37,9 @@ spec:
       - name: url
     name: download
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: /tmp/results.txt
       parameters:
       - name: download-downloaded
         valueFrom:
@@ -85,6 +88,9 @@ spec:
       - name: download-downloaded
     name: get-frequent
     outputs:
+      artifacts:
+      - name: get-frequent-word
+        path: /tmp/message.txt
       parameters:
       - name: get-frequent-word
         valueFrom:

sdk/python/tests/compiler/testdata/default_value.yaml

@@ -57,6 +57,9 @@ spec:
       - name: url
     name: download
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: /tmp/results.txt
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/imagepullsecrets.yaml

@@ -26,6 +26,9 @@ spec:
       - name: message
     name: get-frequent
     outputs:
+      artifacts:
+      - name: get-frequent-word
+        path: /tmp/message.txt
       parameters:
       - name: get-frequent-word
         valueFrom:

sdk/python/tests/compiler/testdata/pipelineparams.yaml

@@ -41,6 +41,9 @@ spec:
       - sh
       - "-c"
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: "/tmp/results.txt"
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml

@@ -25,6 +25,9 @@ spec:
       nodeSelector:
         cloud.google.com/gke-preemptible: 'true'
       outputs:
+        artifacts:
+          - name: flip-output
+            path: /tmp/output
         parameters:
           - name: flip-output
             valueFrom:

sdk/python/tests/compiler/testdata/recursive_do_while.yaml

@@ -33,6 +33,9 @@ spec:
       image: python:alpine3.6
     name: flip
     outputs:
+      artifacts:
+      - name: flip-output
+        path: /tmp/output
       parameters:
       - name: flip-output
         valueFrom:
@@ -47,6 +50,9 @@ spec:
       image: python:alpine3.6
     name: flip-2
     outputs:
+      artifacts:
+      - name: flip-2-output
+        path: /tmp/output
       parameters:
       - name: flip-2-output
         valueFrom:
@@ -61,6 +67,9 @@ spec:
       image: python:alpine3.6
     name: flip-3
     outputs:
+      artifacts:
+      - name: flip-3-output
+        path: /tmp/output
       parameters:
       - name: flip-3-output
         valueFrom:

sdk/python/tests/compiler/testdata/recursive_while.yaml

@@ -55,6 +55,9 @@ spec:
       image: python:alpine3.6
     name: flip
     outputs:
+      artifacts:
+      - name: flip-output
+        path: /tmp/output
       parameters:
       - name: flip-output
         valueFrom:
@@ -69,6 +72,9 @@ spec:
       image: python:alpine3.6
     name: flip-2
     outputs:
+      artifacts:
+      - name: flip-2-output
+        path: /tmp/output
       parameters:
       - name: flip-2-output
         valueFrom:
@@ -83,6 +89,9 @@ spec:
       image: python:alpine3.6
     name: flip-3
     outputs:
+      artifacts:
+      - name: flip-3-output
+        path: /tmp/output
       parameters:
       - name: flip-3-output
         valueFrom:

sdk/python/tests/compiler/testdata/sidecar.yaml

@@ -23,6 +23,9 @@ spec:
     parameters: []
   templates:
   - outputs:
+      artifacts:
+      - name: download-downloaded
+        path: "/tmp/results.txt"
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/tolerations.yaml

@@ -20,6 +20,9 @@ spec:
     parameters: []
   templates:
   - outputs:
+      artifacts:
+      - name: download-downloaded
+        path: "/tmp/results.txt"
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/volume.yaml

@@ -41,6 +41,9 @@ spec:
         name: gcp-credentials
     name: download
     outputs:
+      artifacts:
+      - name: download-downloaded
+        path: /tmp/results.txt
       parameters:
       - name: download-downloaded
         valueFrom:

sdk/python/tests/compiler/testdata/withparam_global.yaml

@@ -46,6 +46,9 @@ spec:
       image: python:alpine3.6
     name: my-out-cop0
     outputs:
+      artifacts:
+      - name: my-out-cop0-out
+        path: /tmp/out.json
       parameters:
       - name: my-out-cop0-out
         valueFrom:

sdk/python/tests/compiler/testdata/withparam_global_dict.yaml

@@ -46,6 +46,9 @@ spec:
       image: python:alpine3.6
     name: my-out-cop0
     outputs:
+      artifacts:
+      - name: my-out-cop0-out
+        path: /tmp/out.json
       parameters:
       - name: my-out-cop0-out
         valueFrom:

sdk/python/tests/compiler/testdata/withparam_output.yaml

@@ -43,6 +43,9 @@ spec:
       image: python:alpine3.6
     name: my-out-cop0
     outputs:
+      artifacts:
+      - name: my-out-cop0-out
+        path: /tmp/out.json
       parameters:
       - name: my-out-cop0-out
         valueFrom:

sdk/python/tests/compiler/testdata/withparam_output_dict.yaml

@@ -43,6 +43,9 @@ spec:
       image: python:alpine3.6
     name: my-out-cop0
     outputs:
+      artifacts:
+      - name: my-out-cop0-out
+        path: /tmp/out.json
       parameters:
       - name: my-out-cop0-out
         valueFrom: