Adds job failure metric

tf-master changed to tf-chief Issue kubeflow#988
krishnadurai · Jun 1, 2019 · 93efa92 · 93efa92
1 parent f140eb6
commit 93efa92
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/docs/monitoring/README.md b/docs/monitoring/README.md
@@ -8,7 +8,7 @@ Currently available metrics to monitor are listed below.
 
 Component Containers:
 * tf-operator
-* tf-master
+* tf-chief
 * tf-ps
 * tf-worker
 
@@ -77,3 +77,8 @@ tf_operator_jobs_deleted
 ```
 tf_operator_jobs_successful
 ```
+
+**Failed Jobs**
+```
+tf_operator_jobs_failed
+```
diff --git a/pkg/controller.v1/tensorflow/status.go b/pkg/controller.v1/tensorflow/status.go
@@ -49,6 +49,13 @@ var (
 	})
 )
 
+var (
+	tfJobsFailureCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "tf_operator_jobs_failed",
+		Help: "Counts number of TF jobs failed",
+	})
+)
+
 // updateStatus updates the status of the tfjob.
 func (tc *TFController) updateStatusSingle(tfjob *tfv1.TFJob, rtype tfv1.TFReplicaType, replicas int, restart, worker0Completed bool) error {
 	tfjobKey, err := KeyFunc(tfjob)
@@ -154,6 +161,7 @@ func (tc *TFController) updateStatusSingle(tfjob *tfv1.TFJob, rtype tfv1.TFRepli
 				tflogger.LoggerForJob(tfjob).Infof("Append tfjob condition error: %v", err)
 				return err
 			}
+			tfJobsFailureCount.Inc()
 		}
 	}
 	return nil