feat: [TKC-2308] test workflow sensitive config parameter (#5721)

* feat: test workflow sensitive config parameter

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: dep update

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: update mapping

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: add sensitive parameter processing

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* feat: add secret expression machine

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: rename env var

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: remove var

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: use test workflow execution secret

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: change reference

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: use options

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: unit test

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: dep update

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: unit test

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: merge

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: merge

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* chore: test

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: escape secret name

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* remove fix

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: template resolving

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: go fmt

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: go fmt

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: unit test

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: enable unit test

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: unit test

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: template unescaping

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: increase iteration

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: dedup code

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: escape dots

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

* fix: dep update

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

---------

Signed-off-by: Vladislav Sukhin <vladislav@kubeshop.io>

api/v1/testkube.yaml

@@ -9451,6 +9451,10 @@ components:
           $ref: "#/components/schemas/BoxedInteger"
         multipleOf:
           $ref: "#/components/schemas/BoxedInteger"
+        sensitive:
+          type: boolean
+          default: false
+          description: whether this value should be stored in the secret
       required:
         - type
 

cmd/api-server/main.go

@@ -561,6 +561,19 @@ func main() {
 		exitOnError("Creating slack loader", err)
 	}
 
+	// TODO: Make granular environment variables, yet backwards compatible
+	secretConfig := testkube.SecretConfig{
+		Prefix:     cfg.SecretCreationPrefix,
+		List:       cfg.EnableSecretsEndpoint,
+		ListAll:    cfg.EnableSecretsEndpoint && cfg.EnableListingAllSecrets,
+		Create:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
+		Modify:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
+		Delete:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
+		AutoCreate: !cfg.DisableSecretCreation,
+	}
+
+	secretManager := secretmanager.New(clientset, secretConfig)
+
 	testWorkflowProcessor := presets.NewOpenSource(inspector)
 	if mode == common.ModeAgent {
 		testWorkflowProcessor = presets.NewPro(inspector)
@@ -576,6 +589,7 @@ func main() {
 		testWorkflowExecutionsClient,
 		testWorkflowsClient,
 		metrics,
+		secretManager,
 		serviceAccountNames,
 		cfg.GlobalWorkflowTemplateName,
 		cfg.TestkubeNamespace,
@@ -589,19 +603,6 @@ func main() {
 
 	go testWorkflowExecutor.Recover(context.Background())
 
-	// TODO: Make granular environment variables, yet backwards compatible
-	secretConfig := testkube.SecretConfig{
-		Prefix:     cfg.SecretCreationPrefix,
-		List:       cfg.EnableSecretsEndpoint,
-		ListAll:    cfg.EnableSecretsEndpoint && cfg.EnableListingAllSecrets,
-		Create:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
-		Modify:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
-		Delete:     cfg.EnableSecretsEndpoint && !cfg.DisableSecretCreation,
-		AutoCreate: !cfg.DisableSecretCreation,
-	}
-
-	secretManager := secretmanager.New(clientset, secretConfig)
-
 	api := apiv1.NewTestkubeAPI(
 		cfg.TestkubeNamespace,
 		resultsRepository,

cmd/tcl/testworkflow-toolkit/commands/parallel.go

@@ -33,6 +33,7 @@ import (
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/expressions"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowcontroller"
+	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/presets"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage"
@@ -202,7 +203,8 @@ func NewParallelCmd() *cobra.Command {
 				// Build the resources bundle
 				scheduledAt := time.Now()
 				bundle, err := presets.NewPro(inspector).
-					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: *spec}, machine, baseMachine, params.MachineAt(index))
+					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: *spec}, testworkflowprocessor.BundleOptions{},
+						machine, baseMachine, params.MachineAt(index))
 				if err != nil {
 					fmt.Printf("%d: failed to prepare resources: %s\n", index, err.Error())
 					registry.Destroy(index)

cmd/tcl/testworkflow-toolkit/commands/services.go

@@ -242,7 +242,8 @@ func NewServicesCmd() *cobra.Command {
 				// Build the resources bundle
 				scheduledAt := time.Now()
 				bundle, err := presets.NewPro(inspector).
-					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: instance.Spec}, machine, baseMachine, params.MachineAt(index))
+					Bundle(context.Background(), &testworkflowsv1.TestWorkflow{Spec: instance.Spec}, testworkflowprocessor.BundleOptions{},
+						machine, baseMachine, params.MachineAt(index))
 				if err != nil {
 					log("error", "failed to build the service", err.Error())
 					return false

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/kubepug/kubepug v1.7.1
-	github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240819101144-e559df091f2b
+	github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d
 	github.com/minio/minio-go/v7 v7.0.47
 	github.com/montanaflynn/stats v0.6.6
 	github.com/moogar0880/problems v0.1.1

go.sum

@@ -371,8 +371,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw=
 github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g=
-github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240819101144-e559df091f2b h1:5DJ+7lhGTAp29loj1qbE36mzpG4xIFwsWQ7tNV4m8q8=
-github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240819101144-e559df091f2b/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
+github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d h1:ZwsRzKUJ+pMC2pF47L/O06Vq8vOYZe17GO3N2rL9lCk=
+github.com/kubeshop/testkube-operator v1.15.2-beta1.0.20240916120302-48b0ceed8c5d/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=

internal/app/api/v1/testworkflows.go

@@ -327,8 +327,17 @@ func (s *TestkubeAPI) PreviewTestWorkflowHandler() fiber.Handler {
 				tplsMap[name] = *tpl
 			}
 
+			// Get information about execution namespace
+			// TODO: Think what to do when it is dynamic - create in all execution namespaces?
+			execNamespace := obj.Namespace
+			if obj.Spec.Job != nil && obj.Spec.Job.Namespace != "" {
+				execNamespace = obj.Spec.Job.Namespace
+			}
+
+			// Handle secrets auto-creation
+			secrets := s.SecretManager.Batch(execNamespace, "tw-", obj.Name)
 			// Resolve the TestWorkflow
-			err = testworkflowresolver.ApplyTemplates(obj, tplsMap)
+			err = testworkflowresolver.ApplyTemplates(obj, tplsMap, secrets.Append)
 			if err != nil {
 				return s.BadRequest(c, errPrefix, "resolving error", err)
 			}

pkg/api/v1/testkube/model_test_workflow_parameter_schema.go

@@ -29,4 +29,6 @@ type TestWorkflowParameterSchema struct {
 	ExclusiveMinimum *BoxedInteger `json:"exclusiveMinimum,omitempty"`
 	ExclusiveMaximum *BoxedInteger `json:"exclusiveMaximum,omitempty"`
 	MultipleOf       *BoxedInteger `json:"multipleOf,omitempty"`
+	// whether this value should be stored in the secret
+	Sensitive bool `json:"sensitive,omitempty"`
 }

pkg/expressions/generic.go

@@ -212,6 +212,11 @@ func resolve(v reflect.Value, t tagData, m []Machine, force bool, finalize bool)
 				vv, _ = expr2.Static().StringValue()
 			} else {
 				vv = expr.Template()
+				if t.value == "template" && !IsTemplateStringWithoutExpressions(str) {
+					if IsTemplateStringWithInternalFnCall(vv) {
+						vv = CleanTemplateStringInternalFnCall(vv)
+					}
+				}
 			}
 			changed = vv != str
 			if ptr.Kind() == reflect.String {

pkg/expressions/libs/secret.go

@@ -0,0 +1,44 @@
+package libs
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/pkg/expressions"
+)
+
+func NewSecretMachine(mapEnvs map[string]corev1.EnvVarSource) expressions.Machine {
+	return expressions.NewMachine().
+		RegisterFunction("secret", func(values ...expressions.StaticValue) (interface{}, bool, error) {
+			if len(values) != 2 {
+				return nil, true, fmt.Errorf(`"secret" function expects 2 arguments, %d provided`, len(values))
+			}
+
+			secretName, _ := values[0].StringValue()
+			keyName, _ := values[1].StringValue()
+			strs := []string{secretName, keyName}
+			for i := range strs {
+				j := 0
+				for _, char := range []string{"-", "."} {
+					for ; strings.Contains(strs[i], char); j++ {
+						strs[i] = strings.Replace(strs[i], char, fmt.Sprintf("_%d_", j), 1)
+					}
+				}
+			}
+
+			envName := fmt.Sprintf("S_N_%s_K_%s", strs[0], strs[1])
+			mapEnvs[envName] = corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: secretName,
+					},
+					Key: keyName,
+				},
+			}
+
+			return expressions.NewValue(fmt.Sprintf("{{%senv.%s}}", expressions.InternalFnCall, envName)), true, nil
+		})
+
+}

pkg/expressions/libs/secret_test.go

@@ -0,0 +1,26 @@
+package libs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/pkg/expressions"
+)
+
+func TestSecret(t *testing.T) {
+	mapEnvs := make(map[string]corev1.EnvVarSource)
+	machine := NewSecretMachine(mapEnvs)
+	assert.Equal(t, "{{"+expressions.InternalFnCall+"env.S_N_name_0_one_1_two_K_key_0_three_1_four}}", MustCall(machine, "secret", "name-one.two", "key-three.four"))
+	assert.EqualValues(t, map[string]corev1.EnvVarSource{
+		"S_N_name_0_one_1_two_K_key_0_three_1_four": {
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: "name-one.two",
+				},
+				Key: "key-three.four",
+			},
+		},
+	}, mapEnvs)
+}

pkg/expressions/parse.go

@@ -255,3 +255,11 @@ func CompileAndResolveTemplate(tpl string, m ...Machine) (Expression, error) {
 func IsTemplateStringWithoutExpressions(tpl string) bool {
 	return !strings.Contains(tpl, "{{")
 }
+
+func IsTemplateStringWithInternalFnCall(tpl string) bool {
+	return strings.Contains(tpl, "{{\"{{\"}}"+InternalFnCall)
+}
+
+func CleanTemplateStringInternalFnCall(tpl string) string {
+	return strings.ReplaceAll(tpl, "{{\"{{\"}}"+InternalFnCall, "{{")
+}

pkg/expressions/stdlib.go

@@ -17,7 +17,8 @@ import (
 )
 
 const (
-	RFC3339Millis = "2006-01-02T15:04:05.000Z07:00"
+	RFC3339Millis  = "2006-01-02T15:04:05.000Z07:00"
+	InternalFnCall = "__internal__fn__call__"
 )
 
 type StdFunction struct {

pkg/mapper/testworkflows/kube_openapi.go

@@ -370,6 +370,7 @@ func MapParameterSchemaKubeToAPI(v testworkflowsv1.ParameterSchema) testkube.Tes
 		ExclusiveMinimum: MapInt64ToBoxedInteger(v.ExclusiveMinimum),
 		ExclusiveMaximum: MapInt64ToBoxedInteger(v.ExclusiveMaximum),
 		MultipleOf:       MapInt64ToBoxedInteger(v.MultipleOf),
+		Sensitive:        v.Sensitive,
 	}
 }
 

pkg/mapper/testworkflows/openapi_kube.go

@@ -269,6 +269,7 @@ func MapParameterSchemaAPIToKube(v testkube.TestWorkflowParameterSchema) testwor
 			ExclusiveMaximum: MapBoxedIntegerToInt64(v.ExclusiveMaximum),
 			MultipleOf:       MapBoxedIntegerToInt64(v.MultipleOf),
 		},
+		Sensitive: v.Sensitive,
 	}
 }
 

pkg/secretmanager/batch.go

@@ -117,3 +117,7 @@ func (s *batch) Create(ctx context.Context, owner *metav1.OwnerReference) error
 	}
 	return nil
 }
+
+func (s *batch) Get() []corev1.Secret {
+	return s.secrets
+}

pkg/testworkflows/testworkflowexecutor/executor.go

@@ -30,6 +30,7 @@ import (
 	testworkflowmappers "github.com/kubeshop/testkube/pkg/mapper/testworkflows"
 	configRepo "github.com/kubeshop/testkube/pkg/repository/config"
 	"github.com/kubeshop/testkube/pkg/repository/testworkflow"
+	"github.com/kubeshop/testkube/pkg/secretmanager"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowcontroller"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor"
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants"
@@ -62,6 +63,7 @@ type executor struct {
 	testWorkflowExecutionsClient   testworkflowsclientv1.TestWorkflowExecutionsInterface
 	testWorkflowsClient            testworkflowsclientv1.Interface
 	metrics                        v1.Metrics
+	secretManager                  secretmanager.SecretManager
 	globalTemplateName             string
 	apiUrl                         string
 	namespace                      string
@@ -83,6 +85,7 @@ func New(emitter *event.Emitter,
 	testWorkflowExecutionsClient testworkflowsclientv1.TestWorkflowExecutionsInterface,
 	testWorkflowsClient testworkflowsclientv1.Interface,
 	metrics v1.Metrics,
+	secretManager secretmanager.SecretManager,
 	serviceAccountNames map[string]string,
 	globalTemplateName, namespace, apiUrl, defaultRegistry string,
 	enableImageDataPersistentCache bool, imageDataPersistentCacheKey, dashboardURI, clusterID string) TestWorkflowExecutor {
@@ -101,6 +104,7 @@ func New(emitter *event.Emitter,
 		testWorkflowExecutionsClient:   testWorkflowExecutionsClient,
 		testWorkflowsClient:            testWorkflowsClient,
 		metrics:                        metrics,
+		secretManager:                  secretManager,
 		serviceAccountNames:            serviceAccountNames,
 		globalTemplateName:             globalTemplateName,
 		apiUrl:                         apiUrl,
@@ -364,14 +368,29 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor
 		}
 	}
 
+	namespace := e.namespace
+	if workflow.Spec.Job != nil && workflow.Spec.Job.Namespace != "" {
+		namespace = workflow.Spec.Job.Namespace
+	}
+
+	if _, ok := e.serviceAccountNames[namespace]; !ok {
+		return execution, fmt.Errorf("not supported execution namespace %s", namespace)
+	}
+
+	// Build the basic Execution data
+	id := primitive.NewObjectID().Hex()
+
+	// Handle secrets auto-creation
+	secrets := e.secretManager.Batch(namespace, "twe-", id)
+
 	// Apply the configuration
-	_, err = testworkflowresolver.ApplyWorkflowConfig(&workflow, testworkflowmappers.MapConfigValueAPIToKube(request.Config))
+	_, err = testworkflowresolver.ApplyWorkflowConfig(&workflow, testworkflowmappers.MapConfigValueAPIToKube(request.Config), secrets.Append)
 	if err != nil {
 		return execution, errors.Wrap(err, "configuration")
 	}
 
 	// Resolve the TestWorkflow
-	err = testworkflowresolver.ApplyTemplates(&workflow, tplsMap)
+	err = testworkflowresolver.ApplyTemplates(&workflow, tplsMap, secrets.Append)
 	if err != nil {
 		return execution, errors.Wrap(err, "resolving error")
 	}
@@ -380,21 +399,12 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor
 	if globalTemplateRef.Name != "" {
 		testworkflowresolver.AddGlobalTemplateRef(&workflow, globalTemplateRef)
 		workflow.Spec.Use = nil
-		err = testworkflowresolver.ApplyTemplates(&workflow, tplsMap)
+		err = testworkflowresolver.ApplyTemplates(&workflow, tplsMap, secrets.Append)
 		if err != nil {
 			return execution, errors.Wrap(err, "resolving with global templates error")
 		}
 	}
 
-	namespace := e.namespace
-	if workflow.Spec.Job != nil && workflow.Spec.Job.Namespace != "" {
-		namespace = workflow.Spec.Job.Namespace
-	}
-
-	if _, ok := e.serviceAccountNames[namespace]; !ok {
-		return execution, fmt.Errorf("not supported execution namespace %s", namespace)
-	}
-
 	// Determine the dashboard information
 	cloudApiKey := common.GetOr(os.Getenv("TESTKUBE_PRO_API_KEY"), os.Getenv("TESTKUBE_CLOUD_API_KEY"))
 	cloudOrgId := common.GetOr(os.Getenv("TESTKUBE_PRO_ORG_ID"), os.Getenv("TESTKUBE_CLOUD_ORG_ID"))
@@ -406,8 +416,6 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor
 			cloudUiUrl, env.Config().Cloud.OrgId, env.Config().Cloud.EnvId)
 	}
 
-	// Build the basic Execution data
-	id := primitive.NewObjectID().Hex()
 	now := time.Now()
 	labels := make(map[string]string)
 	for key, value := range workflow.Labels {
@@ -499,7 +507,7 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor
 	}
 
 	// Validate the TestWorkflow
-	_, err = e.processor.Bundle(ctx, workflow.DeepCopy(), machine, mockExecutionMachine)
+	_, err = e.processor.Bundle(ctx, workflow.DeepCopy(), testworkflowprocessor.BundleOptions{Secrets: secrets.Get()}, machine, mockExecutionMachine)
 	if err != nil {
 		return execution, errors.Wrap(err, "processing error")
 	}
@@ -545,7 +553,7 @@ func (e *executor) Execute(ctx context.Context, workflow testworkflowsv1.TestWor
 	})
 
 	// Process the TestWorkflow
-	bundle, err := e.processor.Bundle(ctx, &workflow, machine, executionMachine)
+	bundle, err := e.processor.Bundle(ctx, &workflow, testworkflowprocessor.BundleOptions{Secrets: secrets.Get()}, machine, executionMachine)
 	if err != nil {
 		return execution, errors.Wrap(err, "processing error")
 	}

pkg/testworkflows/testworkflowprocessor/bundle.go

@@ -16,6 +16,10 @@ import (
 	"github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage"
 )
 
+type BundleOptions struct {
+	Secrets []corev1.Secret
+}
+
 type Bundle struct {
 	Secrets       []corev1.Secret
 	ConfigMaps    []corev1.ConfigMap