Add support for using PipelineRun parameters as matrix parameter values

This commit adds support for using PipelineRun parameters as matrix parameter values by moving the validation for checking matrix parameter types to after parameter substitution. This is necessary for allowing whole array substitutions using the syntax $(params.foo[*]), rather than an explicit YAML array, to be used as a matrix parameter. Prior to this, validation fails because a string referencing an array type was considered a string type and this was done before the applying array replacements.
Note: Whole array replacements for matrix results has not yet been implemented and will be added in a subsequent PR.
This issue is described in more detail here: tektoncd#6056

docs/matrix.md

@@ -213,6 +213,20 @@ tasks:
     - name: param-two
       value: $(params.bar) # array replacement from array param
 ```
+
+`Matrix.Params` supports whole array replacements from array `Parameters`.
+
+```yaml
+tasks:
+...
+- name: task-4
+  taskRef:
+    name: task-4
+  matrix:
+    params:
+    - name: param-one
+      value: $(params.bar[*]) # whole array replacement from array param
+```
 #### Parameters in Matrix.Include.Params
 
 `Matrix.Include.Params` takes string replacements from `Parameters` of type String, Array or Object.

examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-array-references.yaml

@@ -0,0 +1,69 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: platform-browsers
+  annotations:
+    description: |
+      A task that does something cool with platforms and browsers
+spec:
+  params:
+    - name: platform
+      default: ""
+    - name: browser
+      default: ""
+  steps:
+    - name: echo
+      image: alpine
+      script: |
+        echo "$(params.platform) and $(params.browser)"
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  generateName: matrixed-pr-
+spec:
+  serviceAccountName: "default"
+  params:
+    - name: platforms
+      value:
+        - linux
+        - mac
+        - windows
+    - name: browsers
+      value:
+        - chrome
+        - safari
+        - firefox
+  pipelineRef:
+    name: matrixed-pipeline
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: matrixed-pipeline
+spec:
+  params:
+    - name: platforms
+      type: array
+    - name: browsers
+      type: array
+  tasks:
+    - name: platforms-and-browsers
+      matrix:
+        params:
+          - name: platform
+            value: $(params.platforms[*])
+          - name: browser
+            value: $(params.browsers[*])
+      taskRef:
+        name: platform-browsers
+  finally:
+    - name: platforms-and-browsers-in-finally
+      matrix:
+        params:
+          - name: platform
+            value: $(params.platforms[*])
+          - name: browser
+            value: $(params.browsers[*])
+      taskRef:
+        name: platform-browsers

pkg/apis/pipeline/v1/matrix_types.go

@@ -15,8 +15,8 @@ package v1
 
 import (
 	"context"
-	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/tektoncd/pipeline/pkg/apis/config"
 	"golang.org/x/exp/maps"
@@ -300,34 +300,6 @@ func (m *Matrix) validateCombinationsCount(ctx context.Context) (errs *apis.Fiel
 	return errs
 }
 
-// validateParams validates the type of Parameter for Matrix.Params and Matrix.Include.Params
-// Matrix.Params must be of type array. Matrix.Include.Params must be of type string.
-// validateParams also validates Matrix.Params for a unique list of params
-// and a unique list of params in each Matrix.Include.Params specification
-func (m *Matrix) validateParams() (errs *apis.FieldError) {
-	if m != nil {
-		if m.HasInclude() {
-			for i, include := range m.Include {
-				errs = errs.Also(include.Params.validateDuplicateParameters().ViaField(fmt.Sprintf("matrix.include[%d].params", i)))
-				for _, param := range include.Params {
-					if param.Value.Type != ParamTypeString {
-						errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("parameters of type string only are allowed, but got param type %s", string(param.Value.Type)), "").ViaFieldKey("matrix.include.params", param.Name))
-					}
-				}
-			}
-		}
-		if m.HasParams() {
-			errs = errs.Also(m.Params.validateDuplicateParameters().ViaField("matrix.params"))
-			for _, param := range m.Params {
-				if param.Value.Type != ParamTypeArray {
-					errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("parameters of type array only are allowed, but got param type %s", string(param.Value.Type)), "").ViaFieldKey("matrix.params", param.Name))
-				}
-			}
-		}
-	}
-	return errs
-}
-
 // validatePipelineParametersVariablesInMatrixParameters validates all pipeline parameter variables including Matrix.Params and Matrix.Include.Params
 // that may contain the reference(s) to other params to make sure those references are used appropriately.
 func (m *Matrix) validatePipelineParametersVariablesInMatrixParameters(prefix string, paramNames sets.String, arrayParamNames sets.String, objectParamNameKeys map[string][]string) (errs *apis.FieldError) {
@@ -360,3 +332,21 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params []Param) (errs *a
 	}
 	return errs
 }
+
+// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references
+// to entire arrays. This is temporary until whole array replacements for matrix paraemeters are supported.
+// See issue #6056 for more details
+func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) {
+	if m.HasParams() {
+		for i, param := range m.Params {
+			val := param.Value.StringVal
+			expressions, ok := GetVarSubstitutionExpressionsForParam(param)
+			if ok {
+				if LooksLikeContainsResultRefs(expressions) && strings.Contains(val, "[*]") {
+					errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i))
+				}
+			}
+		}
+	}
+	return errs
+}

pkg/apis/pipeline/v1/pipeline_types_test.go

@@ -607,155 +607,42 @@ func TestPipelineTask_ValidateMatrix(t *testing.T) {
 		},
 		wantErrs: apis.ErrMultipleOneOf("matrix[foobar]", "params[foobar]"),
 	}, {
-		name: "parameter duplicated in matrix.include.params and pipelinetask.params",
+		name: "parameters in matrix contain references to arrays",
 		pt: &PipelineTask{
 			Name: "task",
-			Matrix: &Matrix{
-				Include: IncludeParamsList{{
-					Name: "duplicate-param",
-					Params: Params{{
-						Name: "duplicate", Value: ParamValue{Type: ParamTypeString, StringVal: "foo"},
-					}}},
-				}},
-			Params: Params{{
-				Name: "duplicate", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo", "bar"}},
-			}},
-		},
-		wantErrs: apis.ErrMultipleOneOf("matrix[duplicate]", "params[duplicate]"),
-	}, {
-		name: "duplicate parameters in matrix.params",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Params: Params{{
-					Name: "foobar", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo", "bar"}},
-				}, {
-					Name: "foobar", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo-1", "bar-1"}},
-				}}},
-		},
-		wantErrs: &apis.FieldError{
-			Message: `parameter names must be unique, the parameter "foobar" is also defined at`,
-			Paths:   []string{"matrix.params[1].name"},
-		},
-	}, {
-		name: "parameters unique in matrix and params",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Params: Params{{
-					Name: "foobar", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo", "bar"}},
-				}}},
 			Params: Params{{
+				Name: "foobar", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo", "bar"}},
+			}, {
 				Name: "barfoo", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"bar", "foo"}},
 			}},
-		},
-	}, {
-		name: "duplicate parameters in matrix.include.params",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Include: IncludeParamsList{{
-					Name: "invalid-include",
-					Params: Params{{
-						Name: "foobar", Value: ParamValue{Type: ParamTypeString, StringVal: "foo"},
-					}, {
-						Name: "foobar", Value: ParamValue{Type: ParamTypeString, StringVal: "foo-1"},
-					}}},
-				}},
-		},
-		wantErrs: &apis.FieldError{
-			Message: `parameter names must be unique, the parameter "foobar" is also defined at`,
-			Paths:   []string{"matrix.include[0].params[1].name"},
-		},
-	}, {
-		name: "parameters in matrix are strings",
-		pt: &PipelineTask{
-			Name: "task",
 			Matrix: &Matrix{
 				Params: Params{{
-					Name: "foo", Value: ParamValue{Type: ParamTypeString, StringVal: "foo"},
+					Name: "foo", Value: ParamValue{Type: ParamTypeString, StringVal: "$(params.foobar[*])"},
 				}, {
-					Name: "bar", Value: ParamValue{Type: ParamTypeString, StringVal: "bar"},
+					Name: "bar", Value: ParamValue{Type: ParamTypeString, StringVal: "$(params.barfoo[*])"},
 				}}},
 		},
-		wantErrs: &apis.FieldError{
-			Message: "invalid value: parameters of type array only are allowed, but got param type string",
-			Paths:   []string{"matrix.params[foo]", "matrix.params[bar]"},
-		},
 	}, {
-		name: "parameters in matrix are arrays",
+		name: "parameters in matrix contain results references",
 		pt: &PipelineTask{
 			Name: "task",
 			Matrix: &Matrix{
 				Params: Params{{
-					Name: "foobar", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo", "bar"}},
-				}, {
-					Name: "barfoo", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"bar", "foo"}},
-				}}},
-		},
-	}, {
-		name: "parameters in include matrix are strings",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Include: IncludeParamsList{{
-					Name: "build-1",
-					Params: Params{{
-						Name: "IMAGE", Value: ParamValue{Type: ParamTypeString, StringVal: "image-1"},
-					}, {
-						Name: "DOCKERFILE", Value: ParamValue{Type: ParamTypeString, StringVal: "path/to/Dockerfile1"},
-					}}},
-				}},
-		},
-	}, {
-		name: "parameters in include matrix are objects",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Include: IncludeParamsList{{
-					Name: "build-1",
-					Params: Params{{
-						Name: "barfoo", Value: ParamValue{Type: ParamTypeObject, ObjectVal: map[string]string{
-							"url":    "$(params.myObject.non-exist-key)",
-							"commit": "$(params.myString)",
-						}},
-					}, {
-						Name: "foobar", Value: ParamValue{Type: ParamTypeObject, ObjectVal: map[string]string{
-							"url":    "$(params.myObject.non-exist-key)",
-							"commit": "$(params.myString)",
-						}},
-					}},
+					Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}},
 				}}},
 		},
-		wantErrs: &apis.FieldError{
-			Message: "invalid value: parameters of type string only are allowed, but got param type object",
-			Paths:   []string{"matrix.include.params[barfoo]", "matrix.include.params[foobar]"},
-		},
 	}, {
-		name: "parameters in include matrix are arrays",
+		name: "parameters in matrix contain whole array results references",
 		pt: &PipelineTask{
 			Name: "task",
 			Matrix: &Matrix{
-				Include: IncludeParamsList{{
-					Name: "build-1",
-					Params: Params{{
-						Name: "foobar", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"foo", "bar"}},
-					}, {
-						Name: "barfoo", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"bar", "foo"}}}},
+				Params: Params{{
+					Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"},
 				}}},
 		},
 		wantErrs: &apis.FieldError{
-			Message: "invalid value: parameters of type string only are allowed, but got param type array",
-			Paths:   []string{"matrix.include.params[barfoo]", "matrix.include.params[foobar]"},
-		},
-	}, {
-		name: "parameters in matrix contain results references",
-		pt: &PipelineTask{
-			Name: "task",
-			Matrix: &Matrix{
-				Params: Params{{
-					Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}},
-				}}},
+			Message: "matrix parameters cannot contain whole array result references",
+			Paths:   []string{"matrix.params[0]"},
 		},
 	}, {
 		name: "count of combinations of parameters in the matrix exceeds the maximum",

pkg/apis/pipeline/v1/pipeline_validation.go

@@ -145,9 +145,9 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr
 		// when the enable-api-fields feature gate is anything but "alpha".
 		errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields))
 		errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx))
+		errs = errs.Also(pt.Matrix.validateNoWholeArrayResults())
 	}
 	errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params))
-	errs = errs.Also(pt.Matrix.validateParams())
 	return errs
 }