diff --git a/pkg/metrics/eventsqueuemetrics/metrics.go b/pkg/metrics/eventsqueuemetrics/metrics.go new file mode 100644 index 00000000000..b3233f5aef8 --- /dev/null +++ b/pkg/metrics/eventsqueuemetrics/metrics.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Tetragon + +package eventsqueuemetrics + +import ( + "github.com/cilium/tetragon/pkg/metrics/consts" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + eventsQueueReceived = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: consts.MetricsNamespace, + Name: "eventsqueue_stats_received", + Help: "The total number of Tetragon events queue received.", + ConstLabels: nil, + }, nil) + eventsQueueLost = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: consts.MetricsNamespace, + Name: "eventsqueue_stats_lost", + Help: "The total number of Tetragon events queue lost.", + ConstLabels: nil, + }, nil) +) + +func InitMetrics(registry *prometheus.Registry) { + registry.MustRegister(eventsQueueReceived) + registry.MustRegister(eventsQueueLost) +} + +func ReceivedInc() { + eventsQueueReceived.WithLabelValues().Inc() +} + +func LostInc() { + eventsQueueLost.WithLabelValues().Inc() +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2d8dce9ac2d..b9f6de8745d 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -11,6 +11,7 @@ import ( "github.com/cilium/tetragon/pkg/metrics/errormetrics" "github.com/cilium/tetragon/pkg/metrics/eventcachemetrics" "github.com/cilium/tetragon/pkg/metrics/eventmetrics" + "github.com/cilium/tetragon/pkg/metrics/eventsqueuemetrics" "github.com/cilium/tetragon/pkg/metrics/kprobemetrics" "github.com/cilium/tetragon/pkg/metrics/mapmetrics" "github.com/cilium/tetragon/pkg/metrics/opcodemetrics" @@ -34,6 +35,7 @@ func InitAllMetrics(registry *prometheus.Registry) { pfmetrics.InitMetrics(registry) processexecmetrics.InitMetrics(registry) ringbufmetrics.InitMetrics(registry) + eventsqueuemetrics.InitMetrics(registry) syscallmetrics.InitMetrics(registry) watchermetrics.InitMetrics(registry) observer.InitMetrics(registry) diff --git a/pkg/observer/observer.go b/pkg/observer/observer.go index 2e957ff963f..7347916eb32 100644 --- a/pkg/observer/observer.go +++ b/pkg/observer/observer.go @@ -21,6 +21,7 @@ import ( "github.com/cilium/tetragon/pkg/bpf" "github.com/cilium/tetragon/pkg/logger" "github.com/cilium/tetragon/pkg/metrics/errormetrics" + "github.com/cilium/tetragon/pkg/metrics/eventsqueuemetrics" "github.com/cilium/tetragon/pkg/metrics/opcodemetrics" "github.com/cilium/tetragon/pkg/metrics/ringbufmetrics" "github.com/cilium/tetragon/pkg/option" @@ -242,6 +243,7 @@ func (k *Observer) RunEvents(stopCtx context.Context, ready func()) error { case eventsQueue <- &record: default: // eventsQueue channel is full, drop the event + eventsqueuemetrics.LostInc() } k.recvCntr++ ringbufmetrics.PerfEventReceived.Inc() @@ -263,6 +265,7 @@ func (k *Observer) RunEvents(stopCtx context.Context, ready func()) error { select { case event := <-eventsQueue: k.receiveEvent(event.RawSample) + eventsqueuemetrics.ReceivedInc() case <-stopCtx.Done(): k.log.WithError(stopCtx.Err()).Infof("Listening for events completed.") k.log.Debugf("Unprocessed events in RB queue: %d", len(eventsQueue))