metrics: collect the metrics of nydusd events

Collect the metrics of nydus daemon events, including INIT, RUNNING and DIED.

Signed-off-by: Bin Tang <tangbin.bin@bytedance.com>

pkg/daemon/daemon.go

@@ -20,6 +20,7 @@ import (
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/collector"
 	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/erofs"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/mount"
@@ -187,6 +188,7 @@ func (d *Daemon) WaitUntilState(expected types.DaemonState) error {
 				return errors.Errorf("daemon %s is not %s yet, current state %s",
 					d.ID(), expected, state)
 			}
+			collector.CollectDaemonEvent(d.ID(), string(expected))
 
 			return nil
 		},

pkg/daemon/types/types.go

@@ -24,10 +24,12 @@ type DaemonInfo struct {
 }
 
 const (
-	DaemonStateUnknown DaemonState = "UNKNOWN"
-	DaemonStateInit    DaemonState = "INIT"
-	DaemonStateReady   DaemonState = "READY"
-	DaemonStateRunning DaemonState = "RUNNING"
+	DaemonStateUnknown   DaemonState = "UNKNOWN"
+	DaemonStateInit      DaemonState = "INIT"
+	DaemonStateReady     DaemonState = "READY"
+	DaemonStateRunning   DaemonState = "RUNNING"
+	DaemonStateDied      DaemonState = "DIED"
+	DaemonStateDestroyed DaemonState = "DESTROYED"
 )
 
 func (info *DaemonInfo) DaemonState() DaemonState {

pkg/manager/manager.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containerd/nydus-snapshotter/pkg/daemon"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/collector"
 	"github.com/containerd/nydus-snapshotter/pkg/store"
 	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
 )
@@ -506,6 +507,8 @@ func (m *Manager) DestroyDaemon(d *daemon.Daemon) error {
 		log.L.Warnf("Failed to wait for daemon, %v", err)
 	}
 
+	collector.CollectDaemonEvent(d.ID(), string(types.DaemonStateDestroyed))
+
 	return nil
 }
 

pkg/manager/monitor.go

@@ -16,7 +16,9 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/collector"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/retry"
 )
 
@@ -217,6 +219,7 @@ func (m *livenessMonitor) Run() {
 
 				if ev.Events&(unix.EPOLLHUP|unix.EPOLLERR) != 0 {
 					log.L.Warnf("Daemon %s died", target.id)
+					collector.CollectDaemonEvent(target.id, string(types.DaemonStateDied))
 					// Notify subscribers that death event happens
 					target.notifier <- deathEvent{daemonID: target.id, path: target.path}
 				}

pkg/metrics/collector/collector.go

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package collector
+
+import "github.com/containerd/nydus-snapshotter/pkg/daemon/types"
+
+type Collector interface {
+	Collect()
+}
+
+func CollectDaemonEvent(daemonID string, event string) error {
+	GlobalDaemonEventCollector.Collect(daemonID, event)
+	return nil
+}
+
+func CollectFsMetrics(m *types.FsMetrics, imageRef string) error {
+	GlobalFsMetricsCollector.Collect(m, imageRef)
+	return nil
+}

pkg/metrics/collector/daemon.go

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package collector
+
+import (
+	"time"
+
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/data"
+)
+
+type DaemonEventCollector struct{}
+
+var GlobalDaemonEventCollector *DaemonEventCollector
+
+func init() {
+	GlobalDaemonEventCollector = &DaemonEventCollector{}
+}
+
+func (d *DaemonEventCollector) Collect(daemonID string, event string) {
+	data.NydusdEvent.WithLabelValues(daemonID, time.Now().Format("2006-01-02 15:04:05.000"), event).Inc()
+}

pkg/metrics/collector/fs.go

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package collector
+
+import (
+	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/data"
+	"github.com/pkg/errors"
+)
+
+type FsMetricsCollector struct{}
+
+var GlobalFsMetricsCollector *FsMetricsCollector
+
+func init() {
+	GlobalFsMetricsCollector = &FsMetricsCollector{}
+}
+
+func (f *FsMetricsCollector) Collect(m *types.FsMetrics, imageRef string) error {
+	data.ReadCount.WithLabelValues(imageRef).Set(float64(m.DataRead))
+	data.OpenFdCount.WithLabelValues(imageRef).Set(float64(m.NrOpens))
+	data.OpenFdMaxCount.WithLabelValues(imageRef).Set(float64(m.NrMaxOpens))
+	data.LastFopTimestamp.WithLabelValues(imageRef).Set(float64(m.LastFopTp))
+
+	for _, h := range data.MetricHists {
+		o, err := h.ToConstHistogram(m, imageRef)
+		if err != nil {
+			return errors.Wrapf(err, "failed to new const histogram for %s", h.Desc.String())
+		}
+		h.Save(o)
+	}
+
+	return nil
+}

pkg/metrics/data/daemon.go

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package data
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	daemonIDLabel = "daemon_id"
+	timeLabel     = "time"
+	eventLabel    = "event"
+)
+
+var (
+	NydusdEvent = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "nydusd_lifetime_events",
+			Help: "The lifetime events of nydus daemon.",
+		},
+		[]string{daemonIDLabel, timeLabel, eventLabel},
+	)
+)

pkg/metrics/exporter/metrics.go → pkg/metrics/data/fs.go

@@ -1,23 +1,21 @@
 /*
  * Copyright (c) 2021. Alibaba Cloud. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package exporter
+package data
 
 import (
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
-	"github.com/containerd/nydus-snapshotter/pkg/metrics/ttl"
+	mtypes "github.com/containerd/nydus-snapshotter/pkg/metrics/types"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/types/ttl"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
 	imageRefLabel = "image_ref"
-	defaultTTL    = 3 * time.Minute
 )
 
 var (
@@ -28,7 +26,7 @@ var (
 			Help: "Total number read of a nydus fs, in Byte.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 
 	OpenFdCount = ttl.NewGaugeVecWithTTL(
@@ -37,7 +35,7 @@ var (
 			Help: "Number of current open files.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 
 	OpenFdMaxCount = ttl.NewGaugeVecWithTTL(
@@ -46,7 +44,7 @@ var (
 			Help: "Number of max open files.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 
 	LastFopTimestamp = ttl.NewGaugeVecWithTTL(
@@ -55,12 +53,12 @@ var (
 			Help: "Timestamp of last file operation.",
 		},
 		[]string{imageRefLabel},
-		defaultTTL,
+		ttl.DefaultTTL,
 	)
 )
 
 // Fs metric histograms
-var FsMetricHists = []*FsMetricHistogram{
+var MetricHists = []*mtypes.MetricHistogram{
 	{
 		Desc: prometheus.NewDesc(
 			"nydusd_block_count_read_hist",
@@ -81,7 +79,7 @@ var FsMetricHists = []*FsMetricHistogram{
 			[]string{imageRefLabel},
 			prometheus.Labels{},
 		),
-		Buckets: MakeFopBuckets(),
+		Buckets: mtypes.MakeFopBuckets(),
 		GetCounters: func(m *types.FsMetrics) []uint64 {
 			return m.FopHits
 		},
@@ -94,7 +92,7 @@ var FsMetricHists = []*FsMetricHistogram{
 			[]string{imageRefLabel},
 			prometheus.Labels{},
 		),
-		Buckets: MakeFopBuckets(),
+		Buckets: mtypes.MakeFopBuckets(),
 		GetCounters: func(m *types.FsMetrics) []uint64 {
 			return m.FopErrors
 		},

pkg/metrics/exporter/exporter.go

@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package exporter
+
+type Exporter interface {
+	Export()
+}
+
+func FileExport() error {
+	return GlobalFileExporter.Export()
+}

pkg/metrics/exporter/export.go → pkg/metrics/exporter/file.go

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2021. Alibaba Cloud. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -13,21 +14,27 @@ import (
 	"os"
 	"time"
 
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/registry"
 	"github.com/pkg/errors"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
-
-	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 )
 
-type Opt func(*Exporter) error
+type Opt func(*FileExporter) error
 
-type Exporter struct {
+type FileExporter struct {
 	outputFile string
 }
 
+var GlobalFileExporter *FileExporter
+
+func init() {
+	var exp FileExporter
+	GlobalFileExporter = &exp
+}
+
 func WithOutputFile(metricsFile string) Opt {
-	return func(e *Exporter) error {
+	return func(e *FileExporter) error {
 		if metricsFile == "" {
 			return errors.New("metrics file path is empty")
 		}
@@ -40,39 +47,20 @@ func WithOutputFile(metricsFile string) Opt {
 	}
 }
 
-func NewExporter(opts ...Opt) (*Exporter, error) {
-	var exp Exporter
-
+func NewFileExporter(opts ...Opt) error {
 	for _, o := range opts {
-		if err := o(&exp); err != nil {
-			return nil, err
-		}
-	}
-
-	return &exp, nil
-}
-
-func (e *Exporter) ExportFsMetrics(m *types.FsMetrics, imageRef string) error {
-	ReadCount.WithLabelValues(imageRef).Set(float64(m.DataRead))
-	OpenFdCount.WithLabelValues(imageRef).Set(float64(m.NrOpens))
-	OpenFdMaxCount.WithLabelValues(imageRef).Set(float64(m.NrMaxOpens))
-	LastFopTimestamp.WithLabelValues(imageRef).Set(float64(m.LastFopTp))
-
-	for _, h := range FsMetricHists {
-		o, err := h.ToConstHistogram(m, imageRef)
-		if err != nil {
-			return errors.Wrapf(err, "failed to new const histogram for %s", h.Desc.String())
+		if err := o(GlobalFileExporter); err != nil {
+			return err
 		}
-		h.Save(o)
 	}
 
-	return e.output()
+	return nil
 }
 
-func (e *Exporter) output() error {
-	ms, err := Registry.Gather()
+func (e *FileExporter) Export() error {
+	ms, err := registry.Registry.Gather()
 	if err != nil {
-		return errors.Wrap(err, "failed to gather all prometheus collectors")
+		return errors.Wrap(err, "failed to gather all prometheus exporters")
 	}
 	for _, m := range ms {
 		if err := e.exportText(m); err != nil {
@@ -83,7 +71,7 @@ func (e *Exporter) output() error {
 	return nil
 }
 
-func (e *Exporter) exportText(m *dto.MetricFamily) error {
+func (e *FileExporter) exportText(m *dto.MetricFamily) error {
 	var b bytes.Buffer
 
 	enc := expfmt.NewEncoder(&b, expfmt.FmtText)
@@ -102,7 +90,7 @@ func (e *Exporter) exportText(m *dto.MetricFamily) error {
 	return e.writeToFile(string(json))
 }
 
-func (e *Exporter) writeToFile(data string) error {
+func (e *FileExporter) writeToFile(data string) error {
 	f, err := os.OpenFile(e.outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
 		return errors.Wrapf(err, "failed to open metrics file on %s", e.outputFile)