From 3ea2f43d44a857f6be95bfb634efae8e5945c7ea Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Thu, 27 Jun 2024 12:32:52 +0200 Subject: [PATCH 1/3] Add host.name to process metrics --- pkg/internal/export/attributes/attr_defs.go | 9 ++++++- pkg/internal/export/attributes/names/attrs.go | 6 +++-- pkg/internal/export/prom/prom_proc_test.go | 11 +++++---- pkg/internal/infraolly/process/status.go | 4 ++++ pkg/internal/svc/svc.go | 4 ++++ pkg/internal/traces/read_decorator.go | 1 + pkg/internal/traces/read_decorator_test.go | 11 +++++++-- pkg/transform/k8s.go | 2 ++ .../k8s/common/k8s_metrics_testfuncs.go | 24 ++++++++++++------- .../k8s/daemonset/k8s_daemonset_main_test.go | 1 + .../daemonset/k8s_daemonset_z_metrics_test.go | 21 ++++++++++++++++ .../k8s/manifests/06-beyla-daemonset.yml | 15 ++++++++++++ .../k8s/otel/k8s_otel_metrics_test.go | 2 +- test/integration/k8s/prom/k8s_prom_test.go | 2 +- 14 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 test/integration/k8s/daemonset/k8s_daemonset_z_metrics_test.go diff --git a/pkg/internal/export/attributes/attr_defs.go b/pkg/internal/export/attributes/attr_defs.go index 5be23d08e..63e15d88b 100644 --- a/pkg/internal/export/attributes/attr_defs.go +++ b/pkg/internal/export/attributes/attr_defs.go @@ -150,8 +150,15 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup { }, } + // TODO: populate it with host resource attributes in https://opentelemetry.io/docs/specs/semconv/resource/host/ + var hostAttributes = AttrReportGroup{ + Attributes: map[attr.Name]Default{ + attr.HostName: true, + }, + } + var processAttributes = AttrReportGroup{ - SubGroups: []*AttrReportGroup{&appKubeAttributes}, + SubGroups: []*AttrReportGroup{&appKubeAttributes, &hostAttributes}, Attributes: map[attr.Name]Default{ attr.ProcCommand: true, attr.ProcCPUState: true, diff --git a/pkg/internal/export/attributes/names/attrs.go b/pkg/internal/export/attributes/names/attrs.go index 82278ab65..ec03e5095 100644 --- a/pkg/internal/export/attributes/names/attrs.go +++ b/pkg/internal/export/attributes/names/attrs.go @@ -67,7 +67,7 @@ const ( ) // Beyla-specific network attributes -var ( +const ( BeylaIP = Name("beyla.ip") Transport = Name("transport") SrcAddress = Name("src.address") @@ -117,7 +117,7 @@ const ( ) // other beyla-specific attributes -var ( +const ( // TargetInstance is a Prometheus-only attribute. // It will expose the process hostname-pid (or K8s Pod). // It is advised for users that to use relabeling rules to @@ -131,6 +131,8 @@ var ( // attributes, which can't be enabled/disabled by the users ServiceName = Name(semconv.ServiceNameKey) ServiceNamespace = Name(semconv.ServiceNamespaceKey) + + HostName = Name(semconv.HostNameKey) ) // traces related attributes diff --git a/pkg/internal/export/prom/prom_proc_test.go b/pkg/internal/export/prom/prom_proc_test.go index 5eac09a14..c203e356d 100644 --- a/pkg/internal/export/prom/prom_proc_test.go +++ b/pkg/internal/export/prom/prom_proc_test.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/beyla/pkg/internal/export/otel" "github.com/grafana/beyla/pkg/internal/infraolly/process" "github.com/grafana/beyla/pkg/internal/pipe/global" + "github.com/grafana/beyla/pkg/internal/svc" ) func TestProcPrometheusEndpoint_AggregatedMetrics(t *testing.T) { @@ -53,13 +54,13 @@ func TestProcPrometheusEndpoint_AggregatedMetrics(t *testing.T) { // WHEN it receives process metrics metrics <- []*process.Status{ - {Command: "foo", + {Service: &svc.ID{}, Command: "foo", CPUUtilisationWait: 3, CPUUtilisationSystem: 2, CPUUtilisationUser: 1, CPUTimeUserDelta: 30, CPUTimeWaitDelta: 20, CPUTimeSystemDelta: 10, IOReadBytesDelta: 123, IOWriteBytesDelta: 456, NetRcvBytesDelta: 12, NetTxBytesDelta: 34, }, - {Command: "bar", + {Service: &svc.ID{}, Command: "bar", CPUUtilisationWait: 31, CPUUtilisationSystem: 21, CPUUtilisationUser: 11, CPUTimeUserDelta: 301, CPUTimeWaitDelta: 201, CPUTimeSystemDelta: 101, IOReadBytesDelta: 321, IOWriteBytesDelta: 654, @@ -82,7 +83,7 @@ func TestProcPrometheusEndpoint_AggregatedMetrics(t *testing.T) { // AND WHEN new metrics are received metrics <- []*process.Status{ - {Command: "foo", + {Service: &svc.ID{}, Command: "foo", CPUUtilisationWait: 4, CPUUtilisationSystem: 1, CPUUtilisationUser: 2, CPUTimeUserDelta: 3, CPUTimeWaitDelta: 2, CPUTimeSystemDelta: 1, IOReadBytesDelta: 31, IOWriteBytesDelta: 10, @@ -140,7 +141,7 @@ func TestProcPrometheusEndpoint_DisaggregatedMetrics(t *testing.T) { // WHEN it receives process metrics metrics <- []*process.Status{ - {Command: "foo", + {Service: &svc.ID{}, Command: "foo", CPUUtilisationWait: 3, CPUUtilisationSystem: 2, CPUUtilisationUser: 1, CPUTimeUserDelta: 30, CPUTimeWaitDelta: 20, CPUTimeSystemDelta: 10, IOReadBytesDelta: 123, IOWriteBytesDelta: 456, @@ -165,7 +166,7 @@ func TestProcPrometheusEndpoint_DisaggregatedMetrics(t *testing.T) { // AND WHEN new metrics are received metrics <- []*process.Status{ - {Command: "foo", + {Service: &svc.ID{}, Command: "foo", CPUUtilisationWait: 4, CPUUtilisationSystem: 1, CPUUtilisationUser: 2, CPUTimeUserDelta: 3, CPUTimeWaitDelta: 2, CPUTimeSystemDelta: 1, IOReadBytesDelta: 3, IOWriteBytesDelta: 2, diff --git a/pkg/internal/infraolly/process/status.go b/pkg/internal/infraolly/process/status.go index 8795b48fa..00f645e8f 100644 --- a/pkg/internal/infraolly/process/status.go +++ b/pkg/internal/infraolly/process/status.go @@ -70,6 +70,8 @@ func NewStatus(pid int32, svcID *svc.ID) *Status { func OTELGetters(name attr.Name) (attributes.Getter[*Status, attribute.KeyValue], bool) { var g attributes.Getter[*Status, attribute.KeyValue] switch name { + case attr.HostName: + g = func(s *Status) attribute.KeyValue { return attribute.Key(attr.HostName).String(s.Service.HostName) } case attr.ProcCommand: g = func(s *Status) attribute.KeyValue { return attribute.Key(attr.ProcCommand).String(s.Command) } case attr.ProcCommandLine: @@ -111,6 +113,8 @@ func OTELGetters(name attr.Name) (attributes.Getter[*Status, attribute.KeyValue] func PromGetters(name attr.Name) (attributes.Getter[*Status, string], bool) { var g attributes.Getter[*Status, string] switch name { + case attr.HostName: + g = func(s *Status) string { return s.Service.HostName } case attr.ProcCommand: g = func(s *Status) string { return s.Command } case attr.ProcCommandLine: diff --git a/pkg/internal/svc/svc.go b/pkg/internal/svc/svc.go index ea9813d86..fc077ed4b 100644 --- a/pkg/internal/svc/svc.go +++ b/pkg/internal/svc/svc.go @@ -72,6 +72,10 @@ type ID struct { // It is stored here at process discovery time, because it might differ form the // UserPID and HostPID fields of the request.PidInfo struct. ProcPID int32 + + // HostName running the process. It will default to the Beyla host and will be overridden + // by other metadata if available (e.g., Pod Name, Node Name, etc...) + HostName string } func (i *ID) String() string { diff --git a/pkg/internal/traces/read_decorator.go b/pkg/internal/traces/read_decorator.go index f079c4390..68ea8e438 100644 --- a/pkg/internal/traces/read_decorator.go +++ b/pkg/internal/traces/read_decorator.go @@ -120,6 +120,7 @@ func hostNamePIDDecorator(cfg *InstanceIDConfig) decorator { } spans[i].ServiceID.Instance = instanceID spans[i].ServiceID.UID = svc.UID(instanceID) + spans[i].ServiceID.HostName = fullHostName } } } diff --git a/pkg/internal/traces/read_decorator_test.go b/pkg/internal/traces/read_decorator_test.go index f6c843f76..ed317609e 100644 --- a/pkg/internal/traces/read_decorator_test.go +++ b/pkg/internal/traces/read_decorator_test.go @@ -2,6 +2,7 @@ package traces import ( "context" + "os" "testing" "time" @@ -66,9 +67,15 @@ func TestReadDecorator(t *testing.T) { {Path: "/bar", Pid: request.PidInfo{HostPID: 1234}}, } outSpans := testutil.ReadChannel(t, decoratedOutput, testTimeout) + expectedHostName := tc.cfg.InstanceID.OverrideHostname + if expectedHostName == "" { + expectedHostName, _ = os.Hostname() + } assert.Equal(t, []request.Span{ - {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID}, Path: "/foo", Pid: request.PidInfo{HostPID: 1234}}, - {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID}, Path: "/bar", Pid: request.PidInfo{HostPID: 1234}}, + {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID, HostName: expectedHostName}, + Path: "/foo", Pid: request.PidInfo{HostPID: 1234}}, + {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID, HostName: expectedHostName}, + Path: "/bar", Pid: request.PidInfo{HostPID: 1234}}, }, outSpans) }) } diff --git a/pkg/transform/k8s.go b/pkg/transform/k8s.go index a69da4293..235b7ed9f 100644 --- a/pkg/transform/k8s.go +++ b/pkg/transform/k8s.go @@ -109,4 +109,6 @@ func appendMetadata(span *request.Span, info *kube.PodInfo) { span.ServiceID.Metadata[attr.Name(owner.LabelName)] = owner.Name owner = owner.Owner } + // override hostname by the Pod name + span.ServiceID.HostName = info.Name } diff --git a/test/integration/k8s/common/k8s_metrics_testfuncs.go b/test/integration/k8s/common/k8s_metrics_testfuncs.go index 13d929f14..09f392396 100644 --- a/test/integration/k8s/common/k8s_metrics_testfuncs.go +++ b/test/integration/k8s/common/k8s_metrics_testfuncs.go @@ -159,17 +159,23 @@ func FeatureGRPCMetricsDecoration(manifest string) features.Feature { ).Feature() } -func FeatureProcessMetricsDecoration() features.Feature { +func FeatureProcessMetricsDecoration(overrideProperties map[string]string) features.Feature { + properties := map[string]string{ + "k8s_namespace_name": "^default$", + "k8s_node_name": ".+-control-plane$", + "k8s_pod_name": "^testserver-.*", + "k8s_pod_uid": UUIDRegex, + "k8s_pod_start_time": TimeRegex, + "k8s_deployment_name": "^testserver$", + "k8s_replicaset_name": "^testserver-", + } + for k, v := range overrideProperties { + properties[k] = v + } return features.New("Decoration of process metrics"). Assess("all the process metrics from currently instrumented services are properly decorated", - testMetricsDecoration(processMetrics, `{k8s_pod_name=~"testserver-.*"}`, map[string]string{ - "k8s_namespace_name": "^default$", - "k8s_node_name": ".+-control-plane$", - "k8s_pod_uid": UUIDRegex, - "k8s_pod_start_time": TimeRegex, - "k8s_deployment_name": "^testserver$", - "k8s_replicaset_name": "^testserver-", - })).Feature() + testMetricsDecoration(processMetrics, `{k8s_pod_name=~"`+properties["k8s_pod_name"]+`"}`, properties), + ).Feature() } func testMetricsDecoration( diff --git a/test/integration/k8s/daemonset/k8s_daemonset_main_test.go b/test/integration/k8s/daemonset/k8s_daemonset_main_test.go index a09b28a71..48c89c568 100644 --- a/test/integration/k8s/daemonset/k8s_daemonset_main_test.go +++ b/test/integration/k8s/daemonset/k8s_daemonset_main_test.go @@ -46,6 +46,7 @@ func TestMain(m *testing.M) { kube.LocalImage("jaegertracing/all-in-one:latest"), kube.Deploy(k8s.PathManifests+"/01-volumes.yml"), kube.Deploy(k8s.PathManifests+"/01-serviceaccount.yml"), + kube.Deploy(k8s.PathManifests+"/02-prometheus-otelscrape.yml"), kube.Deploy(k8s.PathManifests+"/03-otelcol.yml"), kube.Deploy(k8s.PathManifests+"/04-jaeger.yml"), kube.Deploy(k8s.PathManifests+"/05-uninstrumented-service.yml"), diff --git a/test/integration/k8s/daemonset/k8s_daemonset_z_metrics_test.go b/test/integration/k8s/daemonset/k8s_daemonset_z_metrics_test.go new file mode 100644 index 000000000..727ac19d1 --- /dev/null +++ b/test/integration/k8s/daemonset/k8s_daemonset_z_metrics_test.go @@ -0,0 +1,21 @@ +//go:build integration_k8s + +package otel + +import ( + "testing" + + k8s "github.com/grafana/beyla/test/integration/k8s/common" +) + +// to find process information in the prometheus database, +// we need to make sure that this test is executed after the +// tests in k8s_daemonsset_traces_test.go file +func TestProcessMetrics(t *testing.T) { + cluster.TestEnv().Test(t, k8s.FeatureProcessMetricsDecoration(map[string]string{ + "k8s_deployment_name": "^otherinstance$", + "k8s_replicaset_name": "^otherinstance-.*", + "k8s_pod_name": "^otherinstance-.*", + "host_name": "^otherinstance-.*", + })) +} diff --git a/test/integration/k8s/manifests/06-beyla-daemonset.yml b/test/integration/k8s/manifests/06-beyla-daemonset.yml index 117402a99..7c303991e 100644 --- a/test/integration/k8s/manifests/06-beyla-daemonset.yml +++ b/test/integration/k8s/manifests/06-beyla-daemonset.yml @@ -7,6 +7,19 @@ data: attributes: kubernetes: enable: true + select: + process_cpu_time: + include: ["*"] + process_cpu_utilization: + include: ["*"] + process_memory_usage: + include: ["*"] + process_memory_virtual: + include: ["*"] + process_disk_io: + include: ["*"] + process_network_io: + include: ["*"] print_traces: true log_level: debug discovery: @@ -82,3 +95,5 @@ spec: value: "true" - name: BEYLA_OTEL_METRICS_TTL value: "30m0s" + - name: BEYLA_OTEL_METRICS_FEATURES + value: "application,application_process" \ No newline at end of file diff --git a/test/integration/k8s/otel/k8s_otel_metrics_test.go b/test/integration/k8s/otel/k8s_otel_metrics_test.go index cea64c1e1..78d99a943 100644 --- a/test/integration/k8s/otel/k8s_otel_metrics_test.go +++ b/test/integration/k8s/otel/k8s_otel_metrics_test.go @@ -23,5 +23,5 @@ func TestOTEL_MetricsDecoration_GRPC(t *testing.T) { } func TestOTEL_ProcessMetrics(t *testing.T) { - cluster.TestEnv().Test(t, k8s.FeatureProcessMetricsDecoration()) + cluster.TestEnv().Test(t, k8s.FeatureProcessMetricsDecoration(nil)) } diff --git a/test/integration/k8s/prom/k8s_prom_test.go b/test/integration/k8s/prom/k8s_prom_test.go index 3bf9af1ec..716f8a633 100644 --- a/test/integration/k8s/prom/k8s_prom_test.go +++ b/test/integration/k8s/prom/k8s_prom_test.go @@ -23,5 +23,5 @@ func TestPrometheus_MetricsDecoration_GRPC(t *testing.T) { } func TestPrometheus_ProcessMetrics(t *testing.T) { - cluster.TestEnv().Test(t, k8s.FeatureProcessMetricsDecoration()) + cluster.TestEnv().Test(t, k8s.FeatureProcessMetricsDecoration(nil)) } From b63826e24f019487ec2b8bda8b9a92b56f22c200 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Thu, 27 Jun 2024 12:52:42 +0200 Subject: [PATCH 2/3] Fix read decorator test --- pkg/internal/traces/read_decorator_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/internal/traces/read_decorator_test.go b/pkg/internal/traces/read_decorator_test.go index ed317609e..a9ab58a37 100644 --- a/pkg/internal/traces/read_decorator_test.go +++ b/pkg/internal/traces/read_decorator_test.go @@ -2,7 +2,6 @@ package traces import ( "context" - "os" "testing" "time" @@ -30,21 +29,25 @@ func TestReadDecorator(t *testing.T) { cfg ReadDecorator expectedID string expectedUID svc.UID + expectedHN string } for _, tc := range []testCase{{ desc: "dns", cfg: ReadDecorator{InstanceID: InstanceIDConfig{HostnameDNSResolution: true}}, expectedID: dnsHostname + "-1234", expectedUID: svc.UID(dnsHostname + "-1234"), + expectedHN: dnsHostname, }, { desc: "no-dns", expectedID: localHostname + "-1234", expectedUID: svc.UID(localHostname + "-1234"), + expectedHN: localHostname, }, { desc: "override hostname", cfg: ReadDecorator{InstanceID: InstanceIDConfig{OverrideHostname: "foooo"}}, expectedID: "foooo-1234", expectedUID: "foooo-1234", + expectedHN: "foooo", }, { desc: "override HN", cfg: ReadDecorator{InstanceID: InstanceIDConfig{OverrideInstanceID: "instanceee"}}, @@ -52,6 +55,7 @@ func TestReadDecorator(t *testing.T) { // even if we override instance ID, the UID should be set to a really unique value // (same as the automatic instanceID value) expectedUID: svc.UID(localHostname + "-1234"), + expectedHN: localHostname, }} { t.Run(tc.desc, func(t *testing.T) { cfg := tc.cfg @@ -67,14 +71,10 @@ func TestReadDecorator(t *testing.T) { {Path: "/bar", Pid: request.PidInfo{HostPID: 1234}}, } outSpans := testutil.ReadChannel(t, decoratedOutput, testTimeout) - expectedHostName := tc.cfg.InstanceID.OverrideHostname - if expectedHostName == "" { - expectedHostName, _ = os.Hostname() - } assert.Equal(t, []request.Span{ - {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID, HostName: expectedHostName}, + {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID, HostName: tc.expectedHN}, Path: "/foo", Pid: request.PidInfo{HostPID: 1234}}, - {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID, HostName: expectedHostName}, + {ServiceID: svc.ID{Instance: tc.expectedID, UID: tc.expectedUID, HostName: tc.expectedHN}, Path: "/bar", Pid: request.PidInfo{HostPID: 1234}}, }, outSpans) }) From 7649ec49f04477d1c84ef3b30787b81c43790e87 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Thu, 27 Jun 2024 12:57:06 +0200 Subject: [PATCH 3/3] Fix proclang_darwin.go --- pkg/internal/exec/proclang_darwin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/internal/exec/proclang_darwin.go b/pkg/internal/exec/proclang_darwin.go index e9f28de91..1cd3d9e11 100644 --- a/pkg/internal/exec/proclang_darwin.go +++ b/pkg/internal/exec/proclang_darwin.go @@ -6,7 +6,7 @@ import ( "github.com/grafana/beyla/pkg/internal/svc" ) -func FindProcLanguage(_ int32, _ *elf.File) svc.InstrumentableType { +func FindProcLanguage(_ int32, _ *elf.File, _ string) svc.InstrumentableType { return svc.InstrumentableGeneric }