Add breakpoint to TaskRun Spec

Add step-breakpoint container monitoring failure situations When this API detail is provided in the TaskRun payload, step-breakpoint container is added to the TaskRun Pod which can be used upon the failure of a step. Add -breakpoint_on_failure to entrypointer -breakpoint_on_failure disables exit of container upon failure Complete breakpoint execution if no failure If the TaskRun does not fail, free the step-breakpoint container Add continue script for breakpoint container debug-continue script is added to /tekton/debug which the users can use to mark the step as a success
tektoncd · Mar 25, 2021 · 5636cf3 · 5636cf3
1 parent bad4550
commit 5636cf3
Show file tree

Hide file tree

Showing 10 changed files with 316 additions and 49 deletions.
diff --git a/cmd/entrypoint/main.go b/cmd/entrypoint/main.go
@@ -34,16 +34,20 @@ import (
 )
 
 var (
-	ep              = flag.String("entrypoint", "", "Original specified entrypoint to execute")
-	waitFiles       = flag.String("wait_file", "", "Comma-separated list of paths to wait for")
-	waitFileContent = flag.Bool("wait_file_content", false, "If specified, expect wait_file to have content")
-	postFile        = flag.String("post_file", "", "If specified, file to write upon completion")
-	terminationPath = flag.String("termination_path", "/tekton/termination", "If specified, file to write upon termination")
-	results         = flag.String("results", "", "If specified, list of file names that might contain task results")
-	timeout         = flag.Duration("timeout", time.Duration(0), "If specified, sets timeout for step")
+	ep                  = flag.String("entrypoint", "", "Original specified entrypoint to execute")
+	waitFiles           = flag.String("wait_file", "", "Comma-separated list of paths to wait for")
+	waitFileContent     = flag.Bool("wait_file_content", false, "If specified, expect wait_file to have content")
+	postFile            = flag.String("post_file", "", "If specified, file to write upon completion")
+	terminationPath     = flag.String("termination_path", "/tekton/termination", "If specified, file to write upon termination")
+	results             = flag.String("results", "", "If specified, list of file names that might contain task results")
+	timeout             = flag.Duration("timeout", time.Duration(0), "If specified, sets timeout for step")
+	breakpointOnFailure = flag.Bool("breakpoint_on_failure", false, "If specified, expect steps to not skip on failure")
 )
 
-const defaultWaitPollingInterval = time.Second
+const (
+	defaultWaitPollingInterval = time.Second
+	breakpointExitSuffix = ".breakpointexit"
+	)
 
 func cp(src, dst string) error {
 	s, err := os.Open(src)
@@ -64,6 +68,15 @@ func cp(src, dst string) error {
 	return err
 }
 
+func checkForBreakpointOnFailure(e entrypoint.Entrypointer, breakpointExitPostFile string) {
+	if e.BreakpointOnFailure {
+		if waitErr := e.Waiter.Wait(breakpointExitPostFile, false); waitErr != nil {
+			log.Println("error occurred while waiting for " + breakpointExitPostFile)
+		}
+		os.Exit(0)
+	}
+}
+
 func main() {
 	// Add credential flags originally introduced with our legacy credentials helper
 	// image (creds-init).
@@ -97,17 +110,18 @@ func main() {
 	}
 
 	e := entrypoint.Entrypointer{
-		Entrypoint:      *ep,
-		WaitFiles:       strings.Split(*waitFiles, ","),
-		WaitFileContent: *waitFileContent,
-		PostFile:        *postFile,
-		TerminationPath: *terminationPath,
-		Args:            flag.Args(),
-		Waiter:          &realWaiter{waitPollingInterval: defaultWaitPollingInterval},
-		Runner:          &realRunner{},
-		PostWriter:      &realPostWriter{},
-		Results:         strings.Split(*results, ","),
-		Timeout:         timeout,
+		Entrypoint:          *ep,
+		WaitFiles:           strings.Split(*waitFiles, ","),
+		WaitFileContent:     *waitFileContent,
+		PostFile:            *postFile,
+		TerminationPath:     *terminationPath,
+		Args:                flag.Args(),
+		Waiter:              &realWaiter{waitPollingInterval: defaultWaitPollingInterval},
+		Runner:              &realRunner{},
+		PostWriter:          &realPostWriter{},
+		Results:             strings.Split(*results, ","),
+		Timeout:             timeout,
+		BreakpointOnFailure: *breakpointOnFailure,
 	}
 
 	// Copy any creds injected by the controller into the $HOME directory of the current
@@ -117,6 +131,7 @@ func main() {
 	}
 
 	if err := e.Go(); err != nil {
+		breakpointExitPostFile := e.PostFile + breakpointExitSuffix
 		switch t := err.(type) {
 		case skipError:
 			log.Print("Skipping step because a previous step failed")
@@ -132,11 +147,14 @@ func main() {
 			// in both cases has an ExitStatus() method with the
 			// same signature.
 			if status, ok := t.Sys().(syscall.WaitStatus); ok {
+				checkForBreakpointOnFailure(e, breakpointExitPostFile)
 				os.Exit(status.ExitStatus())
 			}
 			log.Fatalf("Error executing command (ExitError): %v", err)
 		default:
+			checkForBreakpointOnFailure(e, breakpointExitPostFile)
 			log.Fatalf("Error executing command: %v", err)
 		}
 	}
 }
+
diff --git a/cmd/entrypoint/waiter.go b/cmd/entrypoint/waiter.go
@@ -42,6 +42,10 @@ func (rw *realWaiter) Wait(file string, expectContent bool) error {
 		} else if !os.IsNotExist(err) {
 			return fmt.Errorf("waiting for %q: %w", file, err)
 		}
+		// Don't skip step if breakpointOnFailure enabled (to halt execution of the TaskRun)
+		//if _, err := os.Stat(file + ".err"); err == nil && !breakpointOnFailure{
+		//	return skipError("error file present, bail and skip the step")
+		//}
 		if _, err := os.Stat(file + ".err"); err == nil {
 			return skipError("error file present, bail and skip the step")
 		}

diff --git a/examples/v1beta1/taskruns/task-result-with-breakpoint.yaml b/examples/v1beta1/taskruns/task-result-with-breakpoint.yaml
@@ -0,0 +1,39 @@
+apiVersion: tekton.dev/v1beta1
+kind: TaskRun
+metadata:
+  generateName: print-date-
+spec:
+  debug:
+    breakpoint: ["onFailure"] 
+  taskSpec:
+    description: |
+      A simple task that prints the date.
+    results:
+      - name: current-date-unix-timestamp
+        description: The current date in unix timestamp format
+      - name: current-date-human-readable
+        description: The current date in human readable format
+    steps:
+      - name: print-date-unix-timestamp
+        image: bash:latest
+        script: |
+          #!/usr/bin/env bash
+          date +%s | tee /tekton/results/current-date-unix-timestamp
+          exit 1 
+      - name: print-date-human-readable
+        image: bash:latest
+        script: |
+          #!/usr/bin/env bash
+          date | tee /tekton/results/current-date-human-readable
+      - name: print-date-unix-timestamp2
+        image: bash:latest
+        script: |
+          #!/usr/bin/env bash
+          date +%s | tee /tekton/results/current-date-unix-timestamp2
+      - name: print-date-human-readable2
+        image: bash:latest
+        script: |
+          #!/usr/bin/env bash
+          date | tee /tekton/results/current-date-human-readable2
+          exit 1
+
diff --git a/pkg/apis/pipeline/v1beta1/taskrun_types.go b/pkg/apis/pipeline/v1beta1/taskrun_types.go
@@ -42,6 +42,8 @@ var (
 
 // TaskRunSpec defines the desired state of TaskRun
 type TaskRunSpec struct {
+	// +optional
+	Debug *TaskRunDebug `json:"debug,omitempty"`
 	// +optional
 	Params []Param `json:"params,omitempty"`
 	// +optional
@@ -77,6 +79,12 @@ const (
 	TaskRunSpecStatusCancelled = "TaskRunCancelled"
 )
 
+// TaskRunDebug defines the breakpoint config for a particular TaskRun
+type TaskRunDebug struct {
+	// +optional
+	Breakpoint []string `json:"breakpoint ,omitempty"`
+}
+
 // TaskRunInputs holds the input values that this task was invoked with.
 type TaskRunInputs struct {
 	// +optional

diff --git a/pkg/entrypoint/entrypointer.go b/pkg/entrypoint/entrypointer.go
@@ -66,6 +66,8 @@ type Entrypointer struct {
 	Results []string
 	// Timeout is an optional user-specified duration within which the Step must complete
 	Timeout *time.Duration
+	// BreakpointOnFailure helps determine if entrypoint execution needs to adapt debugging requirements
+	BreakpointOnFailure bool
 }
 
 // Waiter encapsulates waiting for files to exist.
@@ -103,7 +105,10 @@ func (e Entrypointer) Go() error {
 		if err := e.Waiter.Wait(f, e.WaitFileContent); err != nil {
 			// An error happened while waiting, so we bail
 			// *but* we write postfile to make next steps bail too.
-			e.WritePostFile(e.PostFile, err)
+			// In case of breakpoint on failure do not write post file.
+			if !e.BreakpointOnFailure {
+				e.WritePostFile(e.PostFile, err)
+			}
 			output = append(output, v1beta1.PipelineResourceResult{
 				Key:        "StartedAt",
 				Value:      time.Now().Format(timeFormat),
@@ -146,7 +151,12 @@ func (e Entrypointer) Go() error {
 	}
 
 	// Write the post file *no matter what*
-	e.WritePostFile(e.PostFile, err)
+	if err != nil && e.BreakpointOnFailure {
+		logger.Info("Skipping writing to PostFile")
+	} else {
+		e.WritePostFile(e.PostFile, err)
+	}
+
 
 	// strings.Split(..) with an empty string returns an array that contains one element, an empty string.
 	// This creates an error when trying to open the result folder as a file.

diff --git a/pkg/pod/entrypoint.go b/pkg/pod/entrypoint.go
@@ -39,15 +39,17 @@ const (
 	mountPoint       = "/tekton/tools"
 	entrypointBinary = mountPoint + "/entrypoint"
 
-	downwardVolumeName     = "tekton-internal-downward"
-	downwardMountPoint     = "/tekton/downward"
-	terminationPath        = "/tekton/termination"
-	downwardMountReadyFile = "ready"
-	readyAnnotation        = "tekton.dev/ready"
-	readyAnnotationValue   = "READY"
+	downwardVolumeName   = "tekton-internal-downward"
+	downwardMountPoint   = "/tekton/downward"
+	terminationPath      = "/tekton/termination"
+	readyFile            = "ready"
+	readyAnnotation      = "tekton.dev/ready"
+	readyAnnotationValue = "READY"
 
 	stepPrefix    = "step-"
 	sidecarPrefix = "sidecar-"
+
+	breakpointOnFailure = "onFailure"
 )
 
 var (
@@ -68,7 +70,7 @@ var (
 		VolumeSource: corev1.VolumeSource{
 			DownwardAPI: &corev1.DownwardAPIVolumeSource{
 				Items: []corev1.DownwardAPIVolumeFile{{
-					Path: downwardMountReadyFile,
+					Path: readyFile,
 					FieldRef: &corev1.ObjectFieldSelector{
 						FieldPath: fmt.Sprintf("metadata.annotations['%s']", readyAnnotation),
 					},
@@ -91,7 +93,7 @@ var (
 // command, we must have fetched the image's ENTRYPOINT before calling this
 // method, using entrypoint_lookup.go.
 // Additionally, Step timeouts are added as entrypoint flag.
-func orderContainers(entrypointImage string, commonExtraEntrypointArgs []string, steps []corev1.Container, taskSpec *v1beta1.TaskSpec) (corev1.Container, []corev1.Container, error) {
+func orderContainers(entrypointImage string, commonExtraEntrypointArgs []string, steps []corev1.Container, taskSpec *v1beta1.TaskSpec, breakpointConfig *v1beta1.TaskRunDebug) (corev1.Container, []corev1.Container, error) {
 	initContainer := corev1.Container{
 		Name:  "place-tools",
 		Image: entrypointImage,
@@ -111,7 +113,7 @@ func orderContainers(entrypointImage string, commonExtraEntrypointArgs []string,
 		case 0:
 			argsForEntrypoint = []string{
 				// First step waits for the Downward volume file.
-				"-wait_file", filepath.Join(downwardMountPoint, downwardMountReadyFile),
+				"-wait_file", filepath.Join(downwardMountPoint, readyFile),
 				"-wait_file_content", // Wait for file contents, not just an empty file.
 				// Start next step.
 				"-post_file", filepath.Join(mountPoint, fmt.Sprintf("%d", i)),
@@ -141,6 +143,18 @@ func orderContainers(entrypointImage string, commonExtraEntrypointArgs []string,
 			args = append(cmd[1:], args...)
 			cmd = []string{cmd[0]}
 		}
+
+		if breakpointConfig != nil && len(breakpointConfig.Breakpoint) > 0 {
+			breakpoints := breakpointConfig.Breakpoint
+			for _, b := range breakpoints{
+				// TODO: Add other breakpoints
+				switch b {
+					case breakpointOnFailure:
+						argsForEntrypoint = append(argsForEntrypoint, "-breakpoint_on_failure")
+				}
+			}
+		}
+
 		argsForEntrypoint = append(argsForEntrypoint, "-entrypoint", cmd[0], "--")
 		argsForEntrypoint = append(argsForEntrypoint, args...)
 

diff --git a/pkg/pod/entrypoint_test.go b/pkg/pod/entrypoint_test.go
@@ -88,7 +88,7 @@ func TestOrderContainers(t *testing.T) {
 		VolumeMounts:           []corev1.VolumeMount{toolsMount},
 		TerminationMessagePath: "/tekton/termination",
 	}}
-	gotInit, got, err := orderContainers(images.EntrypointImage, []string{}, steps, nil)
+	gotInit, got, err := orderContainers(images.EntrypointImage, []string{}, steps, nil, nil)
 	if err != nil {
 		t.Fatalf("orderContainers: %v", err)
 	}
@@ -174,7 +174,7 @@ func TestEntryPointResults(t *testing.T) {
 		VolumeMounts:           []corev1.VolumeMount{toolsMount},
 		TerminationMessagePath: "/tekton/termination",
 	}}
-	_, got, err := orderContainers(images.EntrypointImage, []string{}, steps, &taskSpec)
+	_, got, err := orderContainers(images.EntrypointImage, []string{}, steps, &taskSpec, nil)
 	if err != nil {
 		t.Fatalf("orderContainers: %v", err)
 	}
@@ -214,7 +214,7 @@ func TestEntryPointResultsSingleStep(t *testing.T) {
 		VolumeMounts:           []corev1.VolumeMount{toolsMount, downwardMount},
 		TerminationMessagePath: "/tekton/termination",
 	}}
-	_, got, err := orderContainers(images.EntrypointImage, []string{}, steps, &taskSpec)
+	_, got, err := orderContainers(images.EntrypointImage, []string{}, steps, &taskSpec, nil)
 	if err != nil {
 		t.Fatalf("orderContainers: %v", err)
 	}
@@ -250,7 +250,7 @@ func TestEntryPointSingleResultsSingleStep(t *testing.T) {
 		VolumeMounts:           []corev1.VolumeMount{toolsMount, downwardMount},
 		TerminationMessagePath: "/tekton/termination",
 	}}
-	_, got, err := orderContainers(images.EntrypointImage, []string{}, steps, &taskSpec)
+	_, got, err := orderContainers(images.EntrypointImage, []string{}, steps, &taskSpec, nil)
 	if err != nil {
 		t.Fatalf("orderContainers: %v", err)
 	}

diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go
@@ -19,8 +19,6 @@ package pod
 import (
 	"context"
 	"fmt"
-	"path/filepath"
-
 	"github.com/tektoncd/pipeline/pkg/apis/config"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/pod"
@@ -32,6 +30,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+	"path/filepath"
 )
 
 const (
@@ -123,12 +122,16 @@ func (b *Builder) Build(ctx context.Context, taskRun *v1beta1.TaskRun, taskSpec
 
 	// Convert any steps with Script to command+args.
 	// If any are found, append an init container to initialize scripts.
-	scriptsInit, stepContainers, sidecarContainers := convertScripts(b.Images.ShellImage, steps, taskSpec.Sidecars)
+	scriptsInit, stepContainers, sidecarContainers := convertScripts(b.Images.ShellImage, steps, taskSpec.Sidecars, taskRun.Spec.Debug)
 	if scriptsInit != nil {
 		initContainers = append(initContainers, *scriptsInit)
 		volumes = append(volumes, scriptsVolume)
 	}
 
+	if taskRun.Spec.Debug != nil {
+		volumes = append(volumes, debugScriptsVolume, debugInfoVolume)
+	}
+
 	// Initialize any workingDirs under /workspace.
 	if workingDirInit := workingDirInit(b.Images.ShellImage, stepContainers); workingDirInit != nil {
 		initContainers = append(initContainers, *workingDirInit)
@@ -143,7 +146,7 @@ func (b *Builder) Build(ctx context.Context, taskRun *v1beta1.TaskRun, taskSpec
 	// Rewrite steps with entrypoint binary. Append the entrypoint init
 	// container to place the entrypoint binary. Also add timeout flags
 	// to entrypoint binary.
-	entrypointInit, stepContainers, err := orderContainers(b.Images.EntrypointImage, credEntrypointArgs, stepContainers, &taskSpec)
+	entrypointInit, stepContainers, err := orderContainers(b.Images.EntrypointImage, credEntrypointArgs, stepContainers, &taskSpec, taskRun.Spec.Debug)
 	if err != nil {
 		return nil, err
 	}