From b93581d4432e2842334f9fc739d25cd8fadb42c7 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Tue, 13 Aug 2019 21:37:31 -0400 Subject: [PATCH] Implement step scripts This adds a `Script` field to the `Step` type which, when specified, results in a temporary generated executable script file being invoked containing the specified script contents. The result is an easy-to-use scripting option for users who just want to invoke simple scripts inside containers. Details of generated scripts should be considered an implementation detail, and should not be relied upon by users. --- docs/tasks.md | 63 ++++++++- examples/taskruns/step-script.yaml | 51 ++++++++ pkg/apis/pipeline/images.go | 3 +- pkg/apis/pipeline/v1alpha1/task_types.go | 5 + pkg/apis/pipeline/v1alpha1/task_validation.go | 7 + .../pipeline/v1alpha1/task_validation_test.go | 123 +++++++++++------- pkg/reconciler/taskrun/resources/pod.go | 72 +++++++++- pkg/reconciler/taskrun/resources/pod_test.go | 101 ++++++++++++-- 8 files changed, 368 insertions(+), 57 deletions(-) create mode 100644 examples/taskruns/step-script.yaml diff --git a/docs/tasks.md b/docs/tasks.md index deaaf122799..63874f6c564 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -129,7 +129,7 @@ The `steps` field is required. You define one or more `steps` fields to define the body of a `Task`. If multiple `steps` are defined, they will be executed in the same order as they -are defined, if the `Task` is invoked by a [TaskRun](taskruns.md). +are defined, if the `Task` is invoked by a [`TaskRun`](taskruns.md). Each `steps` in a `Task` must specify a container image that adheres to the [container contract](./container-contract.md). For each of the `steps` fields, @@ -146,6 +146,67 @@ or container images that you define: image in the Task, rather than requesting the sum of all of the container image's resource requests. +#### Step Script + +To simplify executing scripts inside a container, a step can specify a `script`. +If this field is present, the step cannot specify `command` or `args`. + +When specified, a `script` gets invoked as if it were the contents of a file in +the container. Scripts should start with a +[shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) line to declare what +tool should be used to interpret the script. That tool must then also be +available within the step's container. + +This allows you to execute a Bash script, if the image includes `bash`: + +```yaml +steps: +- image: ubuntu # contains bash + script: | + #!/usr/bin/env bash + echo "Hello from Bash!" +``` + +...or to execute a Python script, if the image includes `python`: + +```yaml +steps: +- image: python # contains python + script: | + #!/usr/bin/env python3 + print("Hello from Python!") +``` + +...or to execute a Node script, if the image includes `node`: + +```yaml +steps: +- image: node # contains node + script: | + #!/usr/bin/env node + console.log("Hello from Node!") +``` + +This also simplifies executing script files in the workspace: + +```yaml +steps: +- image: ubuntu + script: | + #!/usr/bin/env bash + /workspace/my-script.sh # provided by an input resource +``` + +...or in the container image: + +```yaml +steps: +- image: my-image # contains /bin/my-binary + script: | + #!/usr/bin/env bash + /bin/my-binary +``` + ### Inputs A `Task` can declare the inputs it needs, which can be either or both of: diff --git a/examples/taskruns/step-script.yaml b/examples/taskruns/step-script.yaml new file mode 100644 index 00000000000..2d493420ac9 --- /dev/null +++ b/examples/taskruns/step-script.yaml @@ -0,0 +1,51 @@ +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + generateName: step-script- +spec: + taskSpec: + steps: + - name: bash + image: ubuntu + env: + - name: FOO + value: foooooooo + script: | + #!/usr/bin/env bash + set -euxo pipefail + echo "Hello from Bash!" + echo FOO is ${FOO} + echo substring is ${FOO:2:4} + for i in {1..10}; do + echo line $i + done + + - name: place-file + image: ubuntu + script: | + #!/usr/bin/env bash + echo "echo Hello from script file" > /workspace/hello + chmod +x /workspace/hello + - name: run-file + image: ubuntu + script: | + #!/usr/bin/env bash + /workspace/hello + + - name: node + image: node + script: | + #!/usr/bin/env node + console.log("Hello from Node!") + + - name: python + image: python + script: | + #!/usr/bin/env python3 + print("Hello from Python!") + + - name: perl + image: perl + script: | + #!/usr/bin/perl + print "Hello from Perl!" diff --git a/pkg/apis/pipeline/images.go b/pkg/apis/pipeline/images.go index 068ee1d1167..4fb5989faf0 100644 --- a/pkg/apis/pipeline/images.go +++ b/pkg/apis/pipeline/images.go @@ -16,7 +16,8 @@ limitations under the License. package pipeline -// Images holds the images reference for a number of container images used accross tektoncd pipelines +// Images holds the images reference for a number of container images used +// across tektoncd pipelines. type Images struct { // EntryPointImage is container image containing our entrypoint binary. EntryPointImage string diff --git a/pkg/apis/pipeline/v1alpha1/task_types.go b/pkg/apis/pipeline/v1alpha1/task_types.go index 4d9bffb7653..fc1c0f28689 100644 --- a/pkg/apis/pipeline/v1alpha1/task_types.go +++ b/pkg/apis/pipeline/v1alpha1/task_types.go @@ -66,6 +66,11 @@ type TaskSpec struct { // provided by Container. type Step struct { corev1.Container + + // Script is the contents of an executable file to execute. + // + // If Script is not empty, the Step cannot have an Command or Args. + Script string `json:"script,omitempty"` } // Check that Task may be validated and defaulted. diff --git a/pkg/apis/pipeline/v1alpha1/task_validation.go b/pkg/apis/pipeline/v1alpha1/task_validation.go index a71036435ab..a4771fc7040 100644 --- a/pkg/apis/pipeline/v1alpha1/task_validation.go +++ b/pkg/apis/pipeline/v1alpha1/task_validation.go @@ -127,6 +127,13 @@ func validateSteps(steps []Step) *apis.FieldError { return apis.ErrMissingField("Image") } + if s.Script != "" && (len(s.Args) > 0 || len(s.Command) > 0) { + return &apis.FieldError{ + Message: "script cannot be used with args or command", + Paths: []string{"script"}, + } + } + if s.Name == "" { continue } diff --git a/pkg/apis/pipeline/v1alpha1/task_validation_test.go b/pkg/apis/pipeline/v1alpha1/task_validation_test.go index 89038f40721..cb0f8218e4c 100644 --- a/pkg/apis/pipeline/v1alpha1/task_validation_test.go +++ b/pkg/apis/pipeline/v1alpha1/task_validation_test.go @@ -192,6 +192,16 @@ func TestTaskSpecValidate(t *testing.T) { Image: "some-image", }, }, + }, { + name: "valid step with script", + fields: fields{ + Steps: []v1alpha1.Step{{ + Container: corev1.Container{ + Image: "my-image", + }, + Script: "hello world", + }}, + }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -263,19 +273,17 @@ func TestTaskSpecValidateError(t *testing.T) { fields: fields{ Inputs: &v1alpha1.Inputs{ Resources: []v1alpha1.TaskResource{validResource}, - Params: []v1alpha1.ParamSpec{ - { - Name: "validparam", - Type: v1alpha1.ParamTypeString, - Description: "parameter", - Default: builder.ArrayOrString("default"), - }, { - Name: "param-with-invalid-type", - Type: "invalidtype", - Description: "invalidtypedesc", - Default: builder.ArrayOrString("default"), - }, - }, + Params: []v1alpha1.ParamSpec{{ + Name: "validparam", + Type: v1alpha1.ParamTypeString, + Description: "parameter", + Default: builder.ArrayOrString("default"), + }, { + Name: "param-with-invalid-type", + Type: "invalidtype", + Description: "invalidtypedesc", + Default: builder.ArrayOrString("default"), + }}, }, Steps: validSteps, }, @@ -288,14 +296,12 @@ func TestTaskSpecValidateError(t *testing.T) { fields: fields{ Inputs: &v1alpha1.Inputs{ Resources: []v1alpha1.TaskResource{validResource}, - Params: []v1alpha1.ParamSpec{ - { - Name: "task", - Type: v1alpha1.ParamTypeArray, - Description: "param", - Default: builder.ArrayOrString("default"), - }, - }, + Params: []v1alpha1.ParamSpec{{ + Name: "task", + Type: v1alpha1.ParamTypeArray, + Description: "param", + Default: builder.ArrayOrString("default"), + }}, }, Steps: validSteps, }, @@ -308,14 +314,12 @@ func TestTaskSpecValidateError(t *testing.T) { fields: fields{ Inputs: &v1alpha1.Inputs{ Resources: []v1alpha1.TaskResource{validResource}, - Params: []v1alpha1.ParamSpec{ - { - Name: "task", - Type: v1alpha1.ParamTypeString, - Description: "param", - Default: builder.ArrayOrString("default", "array"), - }, - }, + Params: []v1alpha1.ParamSpec{{ + Name: "task", + Type: v1alpha1.ParamTypeString, + Description: "param", + Default: builder.ArrayOrString("default", "array"), + }}, }, Steps: validSteps, }, @@ -598,22 +602,51 @@ func TestTaskSpecValidateError(t *testing.T) { Message: `non-existent variable in "$(inputs.params.foo) && $(inputs.params.inexistent)" for step arg[0]`, Paths: []string{"taskspec.steps.arg[0]"}, }, - }, - { - name: "Multiple volumes with same name", - fields: fields{ - Steps: validSteps, - Volumes: []corev1.Volume{{ - Name: "workspace", - }, { - Name: "workspace", - }}, - }, - expectedError: apis.FieldError{ - Message: `multiple volumes with same name "workspace"`, - Paths: []string{"volumes.name"}, - }, - }} + }, { + name: "Multiple volumes with same name", + fields: fields{ + Steps: validSteps, + Volumes: []corev1.Volume{{ + Name: "workspace", + }, { + Name: "workspace", + }}, + }, + expectedError: apis.FieldError{ + Message: `multiple volumes with same name "workspace"`, + Paths: []string{"volumes.name"}, + }, + }, { + name: "step with script and args", + fields: fields{ + Steps: []v1alpha1.Step{{ + Container: corev1.Container{ + Image: "myimage", + Args: []string{"arg"}, + }, + Script: "script", + }}, + }, + expectedError: apis.FieldError{ + Message: "script cannot be used with args or command", + Paths: []string{"steps.script"}, + }, + }, { + name: "step with script and command", + fields: fields{ + Steps: []v1alpha1.Step{{ + Container: corev1.Container{ + Image: "myimage", + Command: []string{"command"}, + }, + Script: "script", + }}, + }, + expectedError: apis.FieldError{ + Message: "script cannot be used with args or command", + Paths: []string{"steps.script"}, + }, + }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ts := &v1alpha1.TaskSpec{ diff --git a/pkg/reconciler/taskrun/resources/pod.go b/pkg/reconciler/taskrun/resources/pod.go index 91437b36751..99deb2b2fb1 100644 --- a/pkg/reconciler/taskrun/resources/pod.go +++ b/pkg/reconciler/taskrun/resources/pod.go @@ -50,6 +50,8 @@ const ( taskRunLabelKey = pipeline.GroupName + pipeline.TaskRunLabelKey ManagedByLabelKey = "app.kubernetes.io/managed-by" ManagedByLabelValue = "tekton-pipelines" + + scriptsDir = "/builder/scripts" ) // These are effectively const, but Go doesn't have such an annotation. @@ -87,6 +89,17 @@ var ( // Random byte reader used for pod name generation. // var for testing. randReader = rand.Reader + + // Volume definition attached to Pods generated from TaskRuns that have + // steps that specify a Script. + scriptsVolume = corev1.Volume{ + Name: "place-scripts", + VolumeSource: emptyVolumeSource, + } + scriptsVolumeMount = corev1.VolumeMount{ + Name: "place-scripts", + MountPath: scriptsDir, + } ) const ( @@ -259,6 +272,16 @@ func MakePod(images pipeline.Images, taskRun *v1alpha1.TaskRun, taskSpec v1alpha maxIndicesByResource := findMaxResourceRequest(taskSpec.Steps, corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourceEphemeralStorage) + placeScripts := false + placeScriptsStep := v1alpha1.Step{Container: corev1.Container{ + Name: names.SimpleNameGenerator.RestrictLengthWithRandomSuffix("place-scripts"), + Image: images.BashNoopImage, + TTY: true, + Command: []string{"/ko-app/bash"}, + Args: []string{"-args", ""}, + VolumeMounts: []corev1.VolumeMount{scriptsVolumeMount}, + }} + for i, s := range taskSpec.Steps { s.Env = append(implicitEnvVars, s.Env...) // TODO(mattmoor): Check that volumeMounts match volumes. @@ -275,6 +298,43 @@ func MakePod(images pipeline.Images, taskRun *v1alpha1.TaskRun, taskSpec v1alpha } } + // If the step specifies a Script, generate and invoke an + // executable script file containing each item in the script. + if s.Script != "" { + placeScripts = true + // Append to the place-scripts script to place the + // script file in a known location in the scripts volume. + tmpFile := filepath.Join(scriptsDir, names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("script-%d", i))) + // NOTE: quotes around 'EOF' are important. Without + // them, ${}s in the file are interpreted as env vars + // and likely end up replaced with empty strings. See + // https://stackoverflow.com/a/27921346 + placeScriptsStep.Args[1] += fmt.Sprintf(`tmpfile="%s" +touch ${tmpfile} && chmod +x ${tmpfile} +cat > ${tmpfile} << 'EOF' +%s +EOF +`, tmpFile, s.Script) + // The entrypoint redirecter has already run on this + // step, so we just need to replace the image's + // entrypoint (if any) with the script to run. + // Validation prevents step args from being passed, but + // just to be careful we'll replace any that survived + // entrypoint redirection here. + + // TODO(jasonhall): It's confusing that entrypoint + // redirection isn't done as part of MakePod, and the + // interaction of these two modifications to container + // args might be confusing to debug in the future. + s.Args = append(s.Args, tmpFile) + for i := 0; i < len(s.Args); i++ { + if s.Args[i] == "-entrypoint" { + s.Args = append(s.Args[:i+1], tmpFile) + } + } + s.VolumeMounts = append(s.VolumeMounts, scriptsVolumeMount) + } + if s.WorkingDir == "" { s.WorkingDir = workspaceDir } @@ -283,7 +343,8 @@ func MakePod(images pipeline.Images, taskRun *v1alpha1.TaskRun, taskSpec v1alpha } else { s.Name = names.SimpleNameGenerator.RestrictLength(fmt.Sprintf("%v%v", containerPrefix, s.Name)) } - // use the container name to add the entrypoint biary as an init container + // use the container name to add the entrypoint binary as an + // init container. if s.Name == names.SimpleNameGenerator.RestrictLength(fmt.Sprintf("%v%v", containerPrefix, entrypoint.InitContainerName)) { initSteps = append(initSteps, s) } else { @@ -291,12 +352,21 @@ func MakePod(images pipeline.Images, taskRun *v1alpha1.TaskRun, taskSpec v1alpha podSteps = append(podSteps, s) } } + // Add podTemplate Volumes to the explicitly declared use volumes volumes := append(taskSpec.Volumes, taskRun.Spec.PodTemplate.Volumes...) // Add our implicit volumes and any volumes needed for secrets to the explicitly // declared user volumes. volumes = append(volumes, implicitVolumes...) volumes = append(volumes, secrets...) + + // Add the volume shared to place a script file, if any step specified + // a script. + if placeScripts { + volumes = append(volumes, scriptsVolume) + initSteps = append(initSteps, placeScriptsStep) + } + if err := v1alpha1.ValidateVolumes(volumes); err != nil { return nil, err } diff --git a/pkg/reconciler/taskrun/resources/pod_test.go b/pkg/reconciler/taskrun/resources/pod_test.go index e2a8b067f93..f4b5724113e 100644 --- a/pkg/reconciler/taskrun/resources/pod_test.go +++ b/pkg/reconciler/taskrun/resources/pod_test.go @@ -427,7 +427,6 @@ func TestMakePod(t *testing.T) { Volumes: implicitVolumes, }, }, { - desc: "additional-sidecar-container", ts: v1alpha1.TaskSpec{ Steps: []v1alpha1.Step{{Container: corev1.Container{ @@ -463,17 +462,101 @@ func TestMakePod(t *testing.T) { corev1.ResourceEphemeralStorage: resource.MustParse("0"), }, }, - }, - { - Name: "sidecar-name", - Image: "sidecar-image", - Resources: corev1.ResourceRequirements{ - Requests: nil, - }, + }, { + Name: "sidecar-name", + Image: "sidecar-image", + Resources: corev1.ResourceRequirements{ + Requests: nil, }, - }, + }}, Volumes: implicitVolumes, }, + }, { + desc: "step with script", + ts: v1alpha1.TaskSpec{ + Steps: []v1alpha1.Step{{ + Container: corev1.Container{ + Name: "one", + Image: "image", + Command: []string{"entrypointer"}, + Args: []string{"wait-file", "out-file", "-entrypoint", "image-entrypoint", "--"}, + }, + Script: "echo hello from step one", + }, { + Container: corev1.Container{ + Name: "two", + Image: "image", + VolumeMounts: []corev1.VolumeMount{{Name: "i-have-a-volume-mount"}}, + Command: []string{"entrypointer"}, + // args aren't valid, but just in case they end up here we'll replace them. + Args: []string{"wait-file", "out-file", "-entrypoint", "image-entrypoint", "--", "args", "somehow"}, + }, + Script: `#!/usr/bin/env python +print("Hello from Python")`, + }}, + }, + want: &corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + InitContainers: []corev1.Container{{ + Name: containerPrefix + credsInit + "-9l9zj", + Image: credsImage, + Command: []string{"/ko-app/creds-init"}, + Args: []string{}, + Env: implicitEnvVars, + VolumeMounts: implicitVolumeMounts, + WorkingDir: workspaceDir, + }, { + Name: "place-scripts-mz4c7", + Image: images.BashNoopImage, + Command: []string{"/ko-app/bash"}, + TTY: true, + Args: []string{"-args", `tmpfile="/builder/scripts/script-0-mssqb" +touch ${tmpfile} && chmod +x ${tmpfile} +cat > ${tmpfile} << 'EOF' +echo hello from step one +EOF +tmpfile="/builder/scripts/script-1-78c5n" +touch ${tmpfile} && chmod +x ${tmpfile} +cat > ${tmpfile} << 'EOF' +#!/usr/bin/env python +print("Hello from Python") +EOF +`}, + VolumeMounts: []corev1.VolumeMount{scriptsVolumeMount}, + }}, + Containers: []corev1.Container{{ + Name: "step-one", + Image: "image", + Command: []string{"entrypointer"}, + Args: []string{"wait-file", "out-file", "-entrypoint", "/builder/scripts/script-0-mssqb"}, + Env: implicitEnvVars, + VolumeMounts: append(implicitVolumeMounts, scriptsVolumeMount), + WorkingDir: workspaceDir, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + corev1.ResourceEphemeralStorage: resource.MustParse("0"), + }, + }, + }, { + Name: "step-two", + Image: "image", + Command: []string{"entrypointer"}, + Args: []string{"wait-file", "out-file", "-entrypoint", "/builder/scripts/script-1-78c5n"}, + Env: implicitEnvVars, + VolumeMounts: append([]corev1.VolumeMount{{Name: "i-have-a-volume-mount"}}, append(implicitVolumeMounts, scriptsVolumeMount)...), + WorkingDir: workspaceDir, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), + corev1.ResourceMemory: resource.MustParse("0"), + corev1.ResourceEphemeralStorage: resource.MustParse("0"), + }, + }, + }}, + Volumes: append(implicitVolumes, scriptsVolume), + }, }} { t.Run(c.desc, func(t *testing.T) { names.TestingSeed()