Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release-v0.56.x] fix: when using remote resources, the related metrics tag name is wrong #7956

Merged
merged 1 commit into from
May 17, 2024
Merged

[release-v0.56.x] fix: when using remote resources, the related metrics tag name is wrong #7956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions pkg/pipelinerunmetrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

"github.com/tektoncd/pipeline/pkg/apis/config"
"github.com/tektoncd/pipeline/pkg/apis/pipeline"
v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
listers "github.com/tektoncd/pipeline/pkg/client/listers/pipeline/v1"
"go.opencensus.io/stats"
Expand Down Expand Up @@ -76,6 +77,8 @@ const (
// ReasonCancelled indicates that a PipelineRun was cancelled.
// Aliased for backwards compatibility; additional reasons should not be added here.
ReasonCancelled = v1.PipelineRunReasonCancelled

anonymous = "anonymous"
)

// Recorder holds keys for Tekton metrics
Expand Down Expand Up @@ -233,6 +236,24 @@ func nilInsertTag(task, taskrun string) []tag.Mutator {
return []tag.Mutator{}
}

func getPipelineTagName(pr *v1.PipelineRun) string {
pipelineName := anonymous
switch {
case pr.Spec.PipelineRef != nil && pr.Spec.PipelineRef.Name != "":
pipelineName = pr.Spec.PipelineRef.Name
case pr.Spec.PipelineSpec != nil:
default:
if len(pr.Labels) > 0 {
pipelineLabel, hasPipelineLabel := pr.Labels[pipeline.PipelineLabelKey]
if hasPipelineLabel && len(pipelineLabel) > 0 {
pipelineName = pipelineLabel
}
}
}

return pipelineName
}

// DurationAndCount logs the duration of PipelineRun execution and
// count for number of PipelineRuns succeed or failed
// returns an error if its failed to log the metrics
Expand Down Expand Up @@ -268,10 +289,8 @@ func (r *Recorder) DurationAndCount(pr *v1.PipelineRun, beforeCondition *apis.Co
}
reason := cond.Reason

pipelineName := "anonymous"
if pr.Spec.PipelineRef != nil && pr.Spec.PipelineRef.Name != "" {
pipelineName = pr.Spec.PipelineRef.Name
}
pipelineName := getPipelineTagName(pr)

ctx, err := tag.New(
context.Background(),
append([]tag.Mutator{tag.Insert(namespaceTag, pr.Namespace),
Expand Down
45 changes: 45 additions & 0 deletions pkg/pipelinerunmetrics/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"testing"
"time"

"github.com/tektoncd/pipeline/pkg/apis/pipeline"

"github.com/tektoncd/pipeline/pkg/apis/config"
v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
fakepipelineruninformer "github.com/tektoncd/pipeline/pkg/client/injection/informers/pipeline/v1/pipelinerun/fake"
Expand Down Expand Up @@ -384,6 +386,49 @@ func TestRecordPipelineRunDurationCount(t *testing.T) {
expectedCount: 1,
beforeCondition: nil,
countWithReason: true,
}, {
name: "for failed pipeline with reference remote pipeline",
pipelineRun: &v1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "pipelinerun-1",
Namespace: "ns",
Labels: map[string]string{
pipeline.PipelineLabelKey: "pipeline-remote",
},
},
Spec: v1.PipelineRunSpec{
PipelineRef: &v1.PipelineRef{
ResolverRef: v1.ResolverRef{
Resolver: "git",
},
},
},
Status: v1.PipelineRunStatus{
Status: duckv1.Status{
Conditions: duckv1.Conditions{{
Type: apis.ConditionSucceeded,
Status: corev1.ConditionFalse,
}},
},
PipelineRunStatusFields: v1.PipelineRunStatusFields{
StartTime: &startTime,
CompletionTime: &completionTime,
},
},
},
expectedDurationTags: map[string]string{
"pipeline": "pipeline-remote",
"pipelinerun": "pipelinerun-1",
"namespace": "ns",
"status": "failed",
},
expectedCountTags: map[string]string{
"status": "failed",
},
expectedDuration: 60,
expectedCount: 1,
beforeCondition: nil,
countWithReason: false,
}} {
t.Run(test.name, func(t *testing.T) {
unregisterMetrics()
Expand Down
28 changes: 20 additions & 8 deletions pkg/taskrunmetrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,24 @@ func nilInsertTag(task, taskrun string) []tag.Mutator {
return []tag.Mutator{}
}

func getTaskTagName(tr *v1.TaskRun) string {
taskName := anonymous
switch {
case tr.Spec.TaskRef != nil && len(tr.Spec.TaskRef.Name) > 0:
taskName = tr.Spec.TaskRef.Name
case tr.Spec.TaskSpec != nil:
default:
if len(tr.Labels) > 0 {
taskLabel, hasTaskLabel := tr.Labels[pipeline.TaskLabelKey]
if hasTaskLabel && len(taskLabel) > 0 {
taskName = taskLabel
}
}
}

return taskName
}

// DurationAndCount logs the duration of TaskRun execution and
// count for number of TaskRuns succeed or failed
// returns an error if its failed to log the metrics
Expand All @@ -329,10 +347,7 @@ func (r *Recorder) DurationAndCount(ctx context.Context, tr *v1.TaskRun, beforeC
duration = tr.Status.CompletionTime.Sub(tr.Status.StartTime.Time)
}

taskName := anonymous
if tr.Spec.TaskRef != nil {
taskName = tr.Spec.TaskRef.Name
}
taskName := getTaskTagName(tr)

cond := tr.Status.GetCondition(apis.ConditionSucceeded)
status := "success"
Expand Down Expand Up @@ -449,10 +464,7 @@ func (r *Recorder) RecordPodLatency(ctx context.Context, pod *corev1.Pod, tr *v1
}

latency := scheduledTime.Sub(pod.CreationTimestamp.Time)
taskName := anonymous
if tr.Spec.TaskRef != nil {
taskName = tr.Spec.TaskRef.Name
}
taskName := getTaskTagName(tr)

ctx, err := tag.New(
ctx,
Expand Down
120 changes: 119 additions & 1 deletion pkg/taskrunmetrics/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,50 @@ func TestRecordTaskRunDurationCount(t *testing.T) {
expectedCount: 1,
beforeCondition: nil,
countWithReason: false,
}, {
name: "for failed taskrun with reference remote task",
taskRun: &v1.TaskRun{
ObjectMeta: metav1.ObjectMeta{
Name: "taskrun-1",
Namespace: "ns",
Labels: map[string]string{
pipeline.TaskLabelKey: "task-remote",
},
},
Spec: v1.TaskRunSpec{
TaskRef: &v1.TaskRef{
ResolverRef: v1.ResolverRef{
Resolver: "git",
},
},
},
Status: v1.TaskRunStatus{
Status: duckv1.Status{
Conditions: duckv1.Conditions{{
Type: apis.ConditionSucceeded,
Status: corev1.ConditionFalse,
}},
},
TaskRunStatusFields: v1.TaskRunStatusFields{
StartTime: &startTime,
CompletionTime: &completionTime,
},
},
},
metricName: "taskrun_duration_seconds",
expectedDurationTags: map[string]string{
"task": "task-remote",
"taskrun": "taskrun-1",
"namespace": "ns",
"status": "failed",
},
expectedCountTags: map[string]string{
"status": "failed",
},
expectedDuration: 60,
expectedCount: 1,
beforeCondition: nil,
countWithReason: false,
}, {
name: "for succeeded taskrun in pipelinerun",
taskRun: &v1.TaskRun{
Expand Down Expand Up @@ -574,12 +618,38 @@ func TestRecordPodLatency(t *testing.T) {
TaskRef: &v1.TaskRef{Name: "task-1"},
},
}
trFromRemoteTask := &v1.TaskRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test-taskrun",
Namespace: "foo",
Labels: map[string]string{
pipeline.TaskLabelKey: "task-remote",
},
},
Spec: v1.TaskRunSpec{
TaskRef: &v1.TaskRef{
ResolverRef: v1.ResolverRef{Resolver: "task-remote"},
},
},
}
emptyLabelTRFromRemoteTask := &v1.TaskRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test-taskrun",
Namespace: "foo",
},
Spec: v1.TaskRunSpec{
TaskRef: &v1.TaskRef{
ResolverRef: v1.ResolverRef{Resolver: "task-remote"},
},
},
}
for _, td := range []struct {
name string
pod *corev1.Pod
expectedTags map[string]string
expectedValue float64
expectingError bool
taskRun *v1.TaskRun
}{{
name: "for scheduled pod",
pod: &corev1.Pod{
Expand All @@ -602,6 +672,53 @@ func TestRecordPodLatency(t *testing.T) {
"namespace": "foo",
},
expectedValue: 4000,
taskRun: taskRun,
}, {
name: "for scheduled pod with reference remote task",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-taskrun-pod-123456",
Namespace: "foo",
CreationTimestamp: creationTime,
},
Status: corev1.PodStatus{
Conditions: []corev1.PodCondition{{
Type: corev1.PodScheduled,
LastTransitionTime: metav1.Time{Time: creationTime.Add(4 * time.Second)},
}},
},
},
expectedTags: map[string]string{
"pod": "test-taskrun-pod-123456",
"task": "task-remote",
"taskrun": "test-taskrun",
"namespace": "foo",
},
expectedValue: 4000,
taskRun: trFromRemoteTask,
}, {
name: "for scheduled pod - empty label tr reference remote task",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-taskrun-pod-123456",
Namespace: "foo",
CreationTimestamp: creationTime,
},
Status: corev1.PodStatus{
Conditions: []corev1.PodCondition{{
Type: corev1.PodScheduled,
LastTransitionTime: metav1.Time{Time: creationTime.Add(4 * time.Second)},
}},
},
},
expectedTags: map[string]string{
"pod": "test-taskrun-pod-123456",
"task": anonymous,
"taskrun": "test-taskrun",
"namespace": "foo",
},
expectedValue: 4000,
taskRun: emptyLabelTRFromRemoteTask,
}, {
name: "for non scheduled pod",
pod: &corev1.Pod{
Expand All @@ -613,6 +730,7 @@ func TestRecordPodLatency(t *testing.T) {
Status: corev1.PodStatus{},
},
expectingError: true,
taskRun: taskRun,
}} {
t.Run(td.name, func(t *testing.T) {
unregisterMetrics()
Expand All @@ -623,7 +741,7 @@ func TestRecordPodLatency(t *testing.T) {
t.Fatalf("NewRecorder: %v", err)
}

if err := metrics.RecordPodLatency(ctx, td.pod, taskRun); td.expectingError && err == nil {
if err := metrics.RecordPodLatency(ctx, td.pod, td.taskRun); td.expectingError && err == nil {
t.Error("RecordPodLatency wanted error, got nil")
} else if !td.expectingError {
if err != nil {
Expand Down