TEP-0106: Support Specifying Metadata per Task in Runtime

While currently Tekton supports specifying metadata for a Task in a Pipeline during "authoring time", specific requirements from a running context lead to this work to offer a flexibility to add metadata during "runtime".

The chosen approach is to add a Metadata field under PipelineTaskRunSpec as for its proper runtime context and easier for a user to follow and utilize, rather than a whole new field. Further consideration and tradeoff can be found in the TEP-0106.

docs/pipelineruns.md

@@ -74,7 +74,7 @@ A `PipelineRun` definition supports the following fields:
   - [`serviceAccountNames`](#mapping-serviceaccount-credentials-to-tasks) - Maps specific `serviceAccountName` values
     to `Tasks` in the `Pipeline`. This overrides the credentials set for the entire `Pipeline`.
   - [`status`](#cancelling-a-pipelinerun) - Specifies options for cancelling a `PipelineRun`. 
-  - [`taskRunSpecs`](#specifying-taskrunspecs) - Specifies a list of `PipelineRunTaskSpec` which allows for setting `ServiceAccountName` and [`Pod` template](./podtemplates.md) for each task. This overrides the `Pod` template set for the entire `Pipeline`.
+  - [`taskRunSpecs`](#specifying-taskrunspecs) - Specifies a list of `PipelineRunTaskSpec` which allows for setting `ServiceAccountName`, [`Pod` template](./podtemplates.md), and `Metadata` for each task. This overrides the `Pod` template set for the entire `Pipeline`.
   - [`timeout`](#configuring-a-failure-timeout) - Specifies the timeout before the `PipelineRun` fails. `timeout` is deprecated and will eventually be removed, so consider using `timeouts` instead.
   - [`timeouts`](#configuring-a-failure-timeout) - Specifies the timeout before the `PipelineRun` fails. `timeouts` allows more granular timeout configuration, at the pipeline, tasks, and finally levels
   - [`podTemplate`](#specifying-a-pod-template) - Specifies a [`Pod` template](./podtemplates.md) to use as the basis for the configuration of the `Pod` that executes each `Task`.
@@ -756,6 +756,39 @@ If used with this `Pipeline`,  `build-task` will use the task specific `PodTempl
 `PipelineTaskRunSpec` may also contain `StepOverrides` and `SidecarOverrides`; see
 [Overriding `Task` `Steps` and `Sidecars`](./taskruns.md#overriding-task-steps-and-sidecars) for more information.
 
+The optional annotations and labels can be added under a `Metadata` field as for a specific running context.
+
+e.g.
+
+Rendering needed secrets with Vault:
+
+```yaml
+spec:
+  pipelineRef:
+    name: pipeline-name
+  taskRunSpecs:
+    - pipelineTaskName: task-name
+      metadata: 
+        annotations:
+          vault.hashicorp.com/agent-inject-secret-foo: "/path/to/foo"
+          vault.hashicorp.com/role: role-name
+```
+
+Updating labels applied in a runtime context:
+
+```yaml
+spec:
+  pipelineRef:
+    name: pipeline-name
+  taskRunSpecs:
+    - pipelineTaskName: task-name
+      metadata: 
+        labels:
+          app: cloudevent
+```
+
+If a metadata key is present in different levels, the value that will be used in the `PipelineRun` is determined using this precedence order: `PipelineRun.spec.taskRunSpec.metadata` > `PipelineRun.metadata` > `Pipeline.spec.tasks.taskSpec.metadata`.
+
 ### Specifying `Workspaces`
 
 If your `Pipeline` specifies one or more `Workspaces`, you must map those `Workspaces` to

pkg/apis/pipeline/v1beta1/pipelinerun_types.go

@@ -600,6 +600,9 @@ type PipelineTaskRunSpec struct {
 	StepOverrides []TaskRunStepOverride `json:"stepOverrides,omitempty"`
 	// +listType=atomic
 	SidecarOverrides []TaskRunSidecarOverride `json:"sidecarOverrides,omitempty"`
+
+	// +optional
+	Metadata PipelineTaskMetadata `json:"metadata,omitempty"`
 }
 
 // GetTaskRunSpec returns the task specific spec for a given
@@ -620,6 +623,7 @@ func (pr *PipelineRun) GetTaskRunSpec(pipelineTaskName string) PipelineTaskRunSp
 			}
 			s.StepOverrides = task.StepOverrides
 			s.SidecarOverrides = task.SidecarOverrides
+			s.Metadata = task.Metadata
 		}
 	}
 	return s

pkg/apis/pipeline/v1beta1/swagger.json

@@ -1609,6 +1609,10 @@
       "description": "PipelineTaskRunSpec  can be used to configure specific specs for a concrete Task",
       "type": "object",
       "properties": {
+        "metadata": {
+          "default": {},
+          "$ref": "#/definitions/v1beta1.PipelineTaskMetadata"
+        },
         "pipelineTaskName": {
           "type": "string"
         },

pkg/reconciler/pipelinerun/pipelinerun.go

@@ -1021,43 +1021,43 @@ func getTaskrunLabels(pr *v1beta1.PipelineRun, pipelineTaskName string, includeP
 }
 
 func combineTaskRunAndTaskSpecLabels(pr *v1beta1.PipelineRun, pipelineTask *v1beta1.PipelineTask) map[string]string {
-	var tsLabels map[string]string
-	trLabels := getTaskrunLabels(pr, pipelineTask.Name, true)
+	labels := make(map[string]string)
+
+	taskRunSpec := pr.GetTaskRunSpec(pipelineTask.Name)
+	addMetadataByPrecedence(labels, taskRunSpec.Metadata.Labels)
+
+	addMetadataByPrecedence(labels, getTaskrunLabels(pr, pipelineTask.Name, true))
 
 	if pipelineTask.TaskSpec != nil {
-		tsLabels = pipelineTask.TaskSpecMetadata().Labels
+		addMetadataByPrecedence(labels, pipelineTask.TaskSpecMetadata().Labels)
 	}
 
-	// labels from TaskRun takes higher precedence over the ones specified in Pipeline through TaskSpec
-	// initialize labels with TaskRun labels
-	labels := trLabels
-	for key, value := range tsLabels {
-		// add labels from TaskSpec if the label does not exist
-		if _, ok := labels[key]; !ok {
-			labels[key] = value
-		}
-	}
 	return labels
 }
 
 func combineTaskRunAndTaskSpecAnnotations(pr *v1beta1.PipelineRun, pipelineTask *v1beta1.PipelineTask) map[string]string {
-	var tsAnnotations map[string]string
-	trAnnotations := getTaskrunAnnotations(pr)
+	annotations := make(map[string]string)
+
+	taskRunSpec := pr.GetTaskRunSpec(pipelineTask.Name)
+	addMetadataByPrecedence(annotations, taskRunSpec.Metadata.Annotations)
+
+	addMetadataByPrecedence(annotations, getTaskrunAnnotations(pr))
 
 	if pipelineTask.TaskSpec != nil {
-		tsAnnotations = pipelineTask.TaskSpecMetadata().Annotations
+		addMetadataByPrecedence(annotations, pipelineTask.TaskSpecMetadata().Annotations)
 	}
 
-	// annotations from TaskRun takes higher precedence over the ones specified in Pipeline through TaskSpec
-	// initialize annotations with TaskRun annotations
-	annotations := trAnnotations
-	for key, value := range tsAnnotations {
-		// add annotations from TaskSpec if the annotation does not exist
-		if _, ok := annotations[key]; !ok {
-			annotations[key] = value
+	return annotations
+}
+
+// addMetadataByPrecedence() adds the elements in addedMetadata to metadata. If the same key is present in both maps, the value from metadata will be used.
+func addMetadataByPrecedence(metadata map[string]string, addedMetadata map[string]string) {
+	for key, value := range addedMetadata {
+		// add new annotations if the key not exists in current ones
+		if _, ok := metadata[key]; !ok {
+			metadata[key] = value
 		}
 	}
-	return annotations
 }
 
 // getFinallyTaskRunTimeout returns the timeout to set when creating the ResolvedPipelineRunTask, which is a finally Task.

pkg/reconciler/pipelinerun/pipelinerun_test.go

@@ -7635,5 +7635,127 @@ spec:
 			}
 		})
 	}
+}
 
+func TestReconcile_PropagatePipelineTaskRunSpecMetadata(t *testing.T) {
+	names.TestingSeed()
+	prName := "test-pipeline-run"
+	ps := []*v1beta1.Pipeline{simpleHelloWorldPipeline}
+	prs := []*v1beta1.PipelineRun{parse.MustParsePipelineRun(t, `
+metadata:
+  name: test-pipeline-run
+  namespace: foo
+spec:
+  pipelineRef:
+    name: test-pipeline
+  taskRunSpecs:
+  - pipelineTaskName: hello-world-1
+    metadata:
+      labels:
+        PipelineTaskRunSpecLabel: PipelineTaskRunSpecValue 	
+      annotations:
+        PipelineTaskRunSpecAnnotation: PipelineTaskRunSpecValue
+    taskServiceAccountName: custom-sa
+`)}
+	ts := []*v1beta1.Task{simpleHelloWorldTask}
+
+	d := test.Data{
+		PipelineRuns: prs,
+		Pipelines:    ps,
+		Tasks:        ts,
+	}
+	prt := newPipelineRunTest(d, t)
+	defer prt.Cancel()
+
+	_, clients := prt.reconcileRun("foo", prName, []string{}, false)
+
+	actual := getTaskRunCreations(t, clients.Pipeline.Actions(), 2)[0]
+	expectedTaskRunObjectMeta := taskRunObjectMeta("test-pipeline-run-hello-world-1", "foo", "test-pipeline-run", "test-pipeline", "hello-world-1", false)
+	expectedTaskRunObjectMeta.Labels["PipelineTaskRunSpecLabel"] = "PipelineTaskRunSpecValue"
+	expectedTaskRunObjectMeta.Annotations["PipelineTaskRunSpecAnnotation"] = "PipelineTaskRunSpecValue"
+	expectedTaskRun := mustParseTaskRunWithObjectMeta(t, expectedTaskRunObjectMeta, `
+spec:
+  resources: {}
+  serviceAccountName: custom-sa
+  taskRef:
+    name: hello-world
+  timeout: 1h0m0s
+`)
+
+	if d := cmp.Diff(actual, expectedTaskRun, ignoreTypeMeta); d != "" {
+		t.Errorf("expected to see propagated metadata from PipelineTaskRunSpec in TaskRun %v created. Diff %s", expectedTaskRun, diff.PrintWantGot(d))
+	}
+}
+
+func TestReconcile_AddMetadataByPrecedence(t *testing.T) {
+	names.TestingSeed()
+	prName := "test-pipeline-run"
+	ps := []*v1beta1.Pipeline{parse.MustParsePipeline(t, `
+metadata:
+  name: test-pipeline
+  namespace: foo
+spec:
+  tasks:
+    - name: hello-world-1
+      taskSpec:
+        steps:
+          - name: foo-step
+            image: foo-image
+        metadata:
+          labels:
+            TestPrecedenceLabel: PipelineTaskSpecValue 
+          annotations:
+            TestPrecedenceAnnotation: PipelineTaskSpecValue
+`)}
+	prs := []*v1beta1.PipelineRun{parse.MustParsePipelineRun(t, `
+metadata:
+  name: test-pipeline-run
+  namespace: foo
+  metadata:
+    labels:
+      TestPrecedenceLabel: PipelineRunValue 
+    annotations:
+      TestPrecedenceAnnotation: PipelineRunValue 
+spec:
+  pipelineRef:
+    name: test-pipeline
+  taskRunSpecs:
+  - pipelineTaskName: hello-world-1
+    metadata:
+      labels:
+        TestPrecedenceLabel: PipelineTaskRunSpecValue 
+      annotations:
+        TestPrecedenceAnnotation: PipelineTaskRunSpecValue
+    taskServiceAccountName: custom-sa
+`)}
+	ts := []*v1beta1.Task{simpleHelloWorldTask}
+
+	d := test.Data{
+		PipelineRuns: prs,
+		Pipelines:    ps,
+		Tasks:        ts,
+	}
+	prt := newPipelineRunTest(d, t)
+	defer prt.Cancel()
+
+	_, clients := prt.reconcileRun("foo", prName, []string{}, false)
+
+	actual := getTaskRunCreations(t, clients.Pipeline.Actions(), 2)[0]
+	expectedTaskRunObjectMeta := taskRunObjectMeta("test-pipeline-run-hello-world-1", "foo", "test-pipeline-run", "test-pipeline", "hello-world-1", false)
+	expectedTaskRunObjectMeta.Labels["TestPrecedenceLabel"] = "PipelineTaskRunSpecValue"
+	expectedTaskRunObjectMeta.Annotations["TestPrecedenceAnnotation"] = "PipelineTaskRunSpecValue"
+	expectedTaskRun := mustParseTaskRunWithObjectMeta(t, expectedTaskRunObjectMeta, `
+spec:
+  resources: {}
+  serviceAccountName: custom-sa
+  taskSpec:
+    steps:
+      - name: foo-step
+        image: foo-image
+  timeout: 1h0m0s
+`)
+
+	if d := cmp.Diff(actual, expectedTaskRun, ignoreTypeMeta); d != "" {
+		t.Errorf("expected to see propagated metadata by the precedence from PipelineTaskRunSpec in TaskRun %v created. Diff %s", expectedTaskRun, diff.PrintWantGot(d))
+	}
 }