Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support to extract init container failure message #5646

Merged
merged 1 commit into from
Feb 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 43 additions & 22 deletions pkg/pod/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,15 +333,20 @@ func updateIncompleteTaskRunStatus(trs *v1beta1.TaskRunStatus, pod *corev1.Pod)

// DidTaskRunFail check the status of pod to decide if related taskrun is failed
func DidTaskRunFail(pod *corev1.Pod) bool {
f := pod.Status.Phase == corev1.PodFailed
if pod.Status.Phase == corev1.PodFailed {
return true
}

for _, s := range pod.Status.ContainerStatuses {
if IsContainerStep(s.Name) {
if s.State.Terminated != nil {
f = f || s.State.Terminated.ExitCode != 0 || isOOMKilled(s)
if s.State.Terminated.ExitCode != 0 || isOOMKilled(s) {
return true
}
}
}
}
return f
return false
}

func areStepsComplete(pod *corev1.Pod) bool {
Expand All @@ -357,26 +362,17 @@ func areStepsComplete(pod *corev1.Pod) bool {
}

func getFailureMessage(logger *zap.SugaredLogger, pod *corev1.Pod) string {
// First, try to surface an error about the actual build step that failed.
// First, try to surface an error about the actual init container that failed.
for _, status := range pod.Status.InitContainerStatuses {
if msg := extractContainerFailureMessage(logger, status, pod.ObjectMeta); len(msg) > 0 {
return fmt.Sprintf("init container failed, %s", msg)
}
}

// Next, try to surface an error about the actual build step that failed.
for _, status := range pod.Status.ContainerStatuses {
term := status.State.Terminated
if term != nil {
msg := status.State.Terminated.Message
r, _ := termination.ParseMessage(logger, msg)
for _, result := range r {
if result.ResultType == v1beta1.InternalTektonResultType && result.Key == "Reason" && result.Value == "TimeoutExceeded" {
// Newline required at end to prevent yaml parser from breaking the log help text at 80 chars
return fmt.Sprintf("%q exited because the step exceeded the specified timeout limit; for logs run: kubectl -n %s logs %s -c %s\n",
status.Name,
pod.Namespace, pod.Name, status.Name)
}
}
if term.ExitCode != 0 {
// Newline required at end to prevent yaml parser from breaking the log help text at 80 chars
return fmt.Sprintf("%q exited with code %d (image: %q); for logs run: kubectl -n %s logs %s -c %s\n",
status.Name, term.ExitCode, status.ImageID,
pod.Namespace, pod.Name, status.Name)
}
if msg := extractContainerFailureMessage(logger, status, pod.ObjectMeta); len(msg) > 0 {
return msg
}
}
// Next, return the Pod's status message if it has one.
Expand All @@ -398,6 +394,31 @@ func getFailureMessage(logger *zap.SugaredLogger, pod *corev1.Pod) string {
return "build failed for unspecified reasons."
}

// extractContainerFailureMessage returns the container failure message by container status or init container status.
func extractContainerFailureMessage(logger *zap.SugaredLogger, status corev1.ContainerStatus, podMetaData metav1.ObjectMeta) string {
term := status.State.Terminated
if term != nil {
msg := status.State.Terminated.Message
r, _ := termination.ParseMessage(logger, msg)
for _, result := range r {
if result.ResultType == v1beta1.InternalTektonResultType && result.Key == "Reason" && result.Value == "TimeoutExceeded" {
// Newline required at end to prevent yaml parser from breaking the log help text at 80 chars
return fmt.Sprintf("%q exited because the step exceeded the specified timeout limit; for logs run: kubectl -n %s logs %s -c %s\n",
status.Name,
podMetaData.Namespace, podMetaData.Name, status.Name)
}
}
if term.ExitCode != 0 {
// Newline required at end to prevent yaml parser from breaking the log help text at 80 chars
return fmt.Sprintf("%q exited with code %d (image: %q); for logs run: kubectl -n %s logs %s -c %s\n",
status.Name, term.ExitCode, status.ImageID,
podMetaData.Namespace, podMetaData.Name, status.Name)
}
}

return ""
}

// IsPodExceedingNodeResources returns true if the Pod's status indicates there
// are insufficient resources to schedule the Pod.
func IsPodExceedingNodeResources(pod *corev1.Pod) bool {
Expand Down
53 changes: 53 additions & 0 deletions pkg/pod/status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1034,6 +1034,59 @@ func TestMakeTaskRunStatus(t *testing.T) {
Sidecars: []v1beta1.SidecarState{},
},
},
}, {
desc: "report init container err message when container from step PodInitializing and init container failed",
pod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod",
Namespace: "foo",
},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{{
Name: "init-A",
}},
Containers: []corev1.Container{{
Name: "step-A",
}},
},
Status: corev1.PodStatus{
Phase: corev1.PodFailed,
InitContainerStatuses: []corev1.ContainerStatus{{
Name: "init-A",
ImageID: "init-image-id-A",
State: corev1.ContainerState{
Terminated: &corev1.ContainerStateTerminated{
ExitCode: 1,
}},
}},
ContainerStatuses: []corev1.ContainerStatus{{
Name: "step-A",
ImageID: "image-id-A",
State: corev1.ContainerState{
Waiting: &corev1.ContainerStateWaiting{
Reason: "PodInitializing",
},
},
}},
},
},
want: v1beta1.TaskRunStatus{
Status: statusFailure(v1beta1.TaskRunReasonFailed.String(), "init container failed, \"init-A\" exited with code 1 (image: \"init-image-id-A\"); for logs run: kubectl -n foo logs pod -c init-A\n"),
TaskRunStatusFields: v1beta1.TaskRunStatusFields{
Steps: []v1beta1.StepState{{
ContainerState: corev1.ContainerState{
Waiting: &corev1.ContainerStateWaiting{
Reason: "PodInitializing",
},
},
Name: "A",
ContainerName: "step-A",
ImageID: "image-id-A",
}},
Sidecars: []v1beta1.SidecarState{},
CompletionTime: &metav1.Time{Time: time.Now()},
},
},
}} {
t.Run(c.desc, func(t *testing.T) {
now := metav1.Now()
Expand Down