grpc: nil check process.Pid

We have had segmentation faults where process.Pid is sometimes nil due to errors in the
cache. Although another patch in this series addresses the current issues, it's still
a bug that tends to crop up from time to time. Add a metric to keep track of these and
perform a nil check so that Tetragon can keep running.

Signed-off-by: William Findlay <will@isovalent.com>

pkg/grpc/exec/exec.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cilium/tetragon/pkg/ktime"
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/metrics/errormetrics"
+	"github.com/cilium/tetragon/pkg/metrics/eventcachemetrics"
 	"github.com/cilium/tetragon/pkg/option"
 	"github.com/cilium/tetragon/pkg/process"
 	readerexec "github.com/cilium/tetragon/pkg/reader/exec"
@@ -67,6 +68,11 @@ func GetProcessExec(event *MsgExecveEventUnix, useCache bool) *tetragon.ProcessE
 		Parent:  tetragonParent,
 	}
 
+	if tetragonProcess.Pid == nil {
+		eventcachemetrics.EventCacheError("GetProcessExec: nil Process.Pid").Inc()
+		return nil
+	}
+
 	if useCache {
 		if ec := eventcache.Get(); ec != nil &&
 			(ec.Needed(tetragonEvent.Process) || (tetragonProcess.Pid.Value > 1 && ec.Needed(tetragonEvent.Parent))) {
@@ -385,6 +391,12 @@ func GetProcessExit(event *MsgExitEventUnix) *tetragon.ProcessExit {
 		Status:  code,
 		Time:    ktime.ToProto(event.Common.Ktime),
 	}
+
+	if tetragonProcess.Pid == nil {
+		eventcachemetrics.EventCacheError("GetProcessExit: nil Process.Pid").Inc()
+		return nil
+	}
+
 	ec := eventcache.Get()
 	if ec != nil &&
 		(ec.Needed(tetragonProcess) ||

pkg/grpc/tracing/tracing.go

@@ -5,6 +5,7 @@ package tracing
 import (
 	"fmt"
 
+	"github.com/cilium/tetragon/pkg/metrics/eventcachemetrics"
 	"github.com/cilium/tetragon/pkg/reader/kernel"
 	"golang.org/x/sys/unix"
 
@@ -304,6 +305,11 @@ func GetProcessKprobe(event *MsgGenericKprobeUnix) *tetragon.ProcessKprobe {
 		PolicyName:   event.PolicyName,
 	}
 
+	if tetragonProcess.Pid == nil {
+		eventcachemetrics.EventCacheError("GetProcessKprobe: nil Process.Pid").Inc()
+		return nil
+	}
+
 	if ec := eventcache.Get(); ec != nil &&
 		(ec.Needed(tetragonProcess) ||
 			(tetragonProcess.Pid.Value > 1 && ec.Needed(tetragonParent))) {
@@ -413,6 +419,11 @@ func (msg *MsgGenericTracepointUnix) HandleMessage() *tetragon.GetEventsResponse
 		Action:     kprobeAction(msg.Action),
 	}
 
+	if tetragonProcess.Pid == nil {
+		eventcachemetrics.EventCacheError("GetProcessTracepoint: nil Process.Pid").Inc()
+		return nil
+	}
+
 	if ec := eventcache.Get(); ec != nil &&
 		(ec.Needed(tetragonProcess) ||
 			(tetragonProcess.Pid.Value > 1 && ec.Needed(tetragonParent))) {
@@ -532,6 +543,11 @@ func GetProcessLoader(msg *MsgProcessLoaderUnix) *tetragon.ProcessLoader {
 		tetragonProcess = process.UnsafeGetProcess()
 	}
 
+	if tetragonProcess.Pid == nil {
+		eventcachemetrics.EventCacheError("GetProcessLoader: nil Process.Pid").Inc()
+		return nil
+	}
+
 	if ec := eventcache.Get(); ec != nil &&
 		(ec.Needed(tetragonProcess) || (tetragonProcess.Pid.Value > 1)) {
 		tetragonEvent := &ProcessLoaderNotify{}
@@ -639,6 +655,11 @@ func GetProcessUprobe(event *MsgGenericUprobeUnix) *tetragon.ProcessUprobe {
 		PolicyName: event.PolicyName,
 	}
 
+	if tetragonProcess.Pid == nil {
+		eventcachemetrics.EventCacheError("GetProcessUprobe: nil Process.Pid").Inc()
+		return nil
+	}
+
 	if ec := eventcache.Get(); ec != nil &&
 		(ec.Needed(tetragonProcess) ||
 			(tetragonProcess.Pid.Value > 1 && ec.Needed(tetragonParent))) {

pkg/metrics/eventcachemetrics/eventcachemetrics.go

@@ -27,12 +27,19 @@ var (
 		Help:        "The total number of Tetragon event cache accesses. For internal use only.",
 		ConstLabels: nil,
 	})
+	eventCacheErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   consts.MetricsNamespace,
+		Name:        "event_cache_errors_total",
+		Help:        "The total of errors encountered while fetching process exec information from the cache.",
+		ConstLabels: nil,
+	}, []string{"error"})
 )
 
 func InitMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(processInfoErrors)
 	registry.MustRegister(podInfoErrors)
 	registry.MustRegister(EventCacheCount)
+	registry.MustRegister(eventCacheErrorsTotal)
 }
 
 // Get a new handle on an processInfoErrors metric for an eventType
@@ -44,3 +51,8 @@ func ProcessInfoError(eventType string) prometheus.Counter {
 func PodInfoError(eventType string) prometheus.Counter {
 	return podInfoErrors.WithLabelValues(eventType)
 }
+
+// Get a new handle on an processInfoErrors metric for an eventType
+func EventCacheError(err string) prometheus.Counter {
+	return eventCacheErrorsTotal.WithLabelValues(err)
+}