From 697093e3897c519599ddca5b5e23db9226dc1a6f Mon Sep 17 00:00:00 2001 From: Raphael Philipe Mendes da Silva Date: Wed, 10 Jan 2024 14:48:13 -0300 Subject: [PATCH] [exporter/awscloudwatchlogs] Feat: include scope in log record of cloudwatchlogs exporter (#30316) **Description:** Include the instrumentation scope in the log records exported by the cloudwatchlogs expoter **Link to tracking Issue:** #29884 **Testing:** Unit tests were added. --------- Signed-off-by: Raphael Silva Co-authored-by: bryan-aguilar <46550959+bryan-aguilar@users.noreply.github.com> --- .../add_scope_awscloudwatchlogsexporter.yaml | 27 ++++++++++ .../awscloudwatchlogsexporter/exporter.go | 40 ++++++++++---- .../exporter_test.go | 53 ++++++++++++++++--- 3 files changed, 103 insertions(+), 17 deletions(-) create mode 100755 .chloggen/add_scope_awscloudwatchlogsexporter.yaml diff --git a/.chloggen/add_scope_awscloudwatchlogsexporter.yaml b/.chloggen/add_scope_awscloudwatchlogsexporter.yaml new file mode 100755 index 000000000000..6522bd10eb70 --- /dev/null +++ b/.chloggen/add_scope_awscloudwatchlogsexporter.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: awscloudwatchlogsexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add instrumentation scope in log records exported to CloudWatch logs + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [30316, 29884] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/awscloudwatchlogsexporter/exporter.go b/exporter/awscloudwatchlogsexporter/exporter.go index 839ee2215730..51c78ca59fc8 100644 --- a/exporter/awscloudwatchlogsexporter/exporter.go +++ b/exporter/awscloudwatchlogsexporter/exporter.go @@ -140,10 +140,11 @@ func pushLogsToCWLogs(logger *zap.Logger, ld plog.Logs, config *Config, pusher c sls := rl.ScopeLogs() for j := 0; j < sls.Len(); j++ { sl := sls.At(j) + scope := sl.Scope() logs := sl.LogRecords() for k := 0; k < logs.Len(); k++ { log := logs.At(k) - event, err := logToCWLog(resourceAttrs, log, config) + event, err := logToCWLog(resourceAttrs, scope, log, config) if err != nil { logger.Debug("Failed to convert to CloudWatch Log", zap.Error(err)) } else { @@ -159,19 +160,26 @@ func pushLogsToCWLogs(logger *zap.Logger, ld plog.Logs, config *Config, pusher c return errs } +type scopeCwLogBody struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Attributes map[string]any `json:"attributes,omitempty"` +} + type cwLogBody struct { - Body any `json:"body,omitempty"` - SeverityNumber int32 `json:"severity_number,omitempty"` - SeverityText string `json:"severity_text,omitempty"` - DroppedAttributesCount uint32 `json:"dropped_attributes_count,omitempty"` - Flags uint32 `json:"flags,omitempty"` - TraceID string `json:"trace_id,omitempty"` - SpanID string `json:"span_id,omitempty"` - Attributes map[string]any `json:"attributes,omitempty"` - Resource map[string]any `json:"resource,omitempty"` + Body any `json:"body,omitempty"` + SeverityNumber int32 `json:"severity_number,omitempty"` + SeverityText string `json:"severity_text,omitempty"` + DroppedAttributesCount uint32 `json:"dropped_attributes_count,omitempty"` + Flags uint32 `json:"flags,omitempty"` + TraceID string `json:"trace_id,omitempty"` + SpanID string `json:"span_id,omitempty"` + Attributes map[string]any `json:"attributes,omitempty"` + Scope *scopeCwLogBody `json:"scope,omitempty"` + Resource map[string]any `json:"resource,omitempty"` } -func logToCWLog(resourceAttrs map[string]any, log plog.LogRecord, config *Config) (*cwlogs.Event, error) { +func logToCWLog(resourceAttrs map[string]any, scope pcommon.InstrumentationScope, log plog.LogRecord, config *Config) (*cwlogs.Event, error) { // TODO(jbd): Benchmark and improve the allocations. // Evaluate go.elastic.co/fastjson as a replacement for encoding/json. logGroupName := config.LogGroupName @@ -214,6 +222,16 @@ func logToCWLog(resourceAttrs map[string]any, log plog.LogRecord, config *Config body.Attributes = attrsValue(log.Attributes()) body.Resource = resourceAttrs + // scope should have a name at least + if scope.Name() != "" { + scopeBody := &scopeCwLogBody{ + Name: scope.Name(), + Version: scope.Version(), + Attributes: attrsValue(scope.Attributes()), + } + body.Scope = scopeBody + } + bodyJSON, err = json.Marshal(body) if err != nil { return &cwlogs.Event{}, err diff --git a/exporter/awscloudwatchlogsexporter/exporter_test.go b/exporter/awscloudwatchlogsexporter/exporter_test.go index 33c1d0f78251..e9c99a216d92 100644 --- a/exporter/awscloudwatchlogsexporter/exporter_test.go +++ b/exporter/awscloudwatchlogsexporter/exporter_test.go @@ -47,6 +47,7 @@ func TestLogToCWLog(t *testing.T) { tests := []struct { name string resource pcommon.Resource + scope pcommon.InstrumentationScope log plog.LogRecord config *Config want cwlogs.Event @@ -56,12 +57,13 @@ func TestLogToCWLog(t *testing.T) { name: "basic", resource: testResource(), log: testLogRecord(), + scope: testScope(), config: &Config{}, want: cwlogs.Event{ GeneratedTime: time.Now(), InputLogEvent: &cloudwatchlogs.InputLogEvent{ Timestamp: aws.Int64(1609719139), - Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"flags":1,"trace_id":"0102030405060708090a0b0c0d0e0f10","span_id":"0102030405060708","attributes":{"key1":1,"key2":"attr2"},"resource":{"host":"abc123","node":5}}`), + Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"flags":1,"trace_id":"0102030405060708090a0b0c0d0e0f10","span_id":"0102030405060708","attributes":{"key1":1,"key2":"attr2"},"scope":{"name":"test-scope","version":"1.0.0","attributes":{"scope-attr":"value"}},"resource":{"host":"abc123","node":5}}`), }, StreamKey: cwlogs.StreamKey{ LogGroupName: "", @@ -72,13 +74,32 @@ func TestLogToCWLog(t *testing.T) { { name: "no resource", resource: pcommon.NewResource(), + scope: testScope(), log: testLogRecord(), config: &Config{}, want: cwlogs.Event{ GeneratedTime: time.Now(), InputLogEvent: &cloudwatchlogs.InputLogEvent{ Timestamp: aws.Int64(1609719139), - Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"flags":1,"trace_id":"0102030405060708090a0b0c0d0e0f10","span_id":"0102030405060708","attributes":{"key1":1,"key2":"attr2"}}`), + Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"flags":1,"trace_id":"0102030405060708090a0b0c0d0e0f10","span_id":"0102030405060708","attributes":{"key1":1,"key2":"attr2"},"scope":{"name":"test-scope","version":"1.0.0","attributes":{"scope-attr":"value"}}}`), + }, + StreamKey: cwlogs.StreamKey{ + LogGroupName: "", + LogStreamName: "", + }, + }, + }, + { + name: "no scope", + resource: testResource(), + log: testLogRecord(), + scope: emptyScope(), + config: &Config{}, + want: cwlogs.Event{ + GeneratedTime: time.Now(), + InputLogEvent: &cloudwatchlogs.InputLogEvent{ + Timestamp: aws.Int64(1609719139), + Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"flags":1,"trace_id":"0102030405060708090a0b0c0d0e0f10","span_id":"0102030405060708","attributes":{"key1":1,"key2":"attr2"},"resource":{"host":"abc123","node":5}}`), }, StreamKey: cwlogs.StreamKey{ LogGroupName: "", @@ -89,6 +110,7 @@ func TestLogToCWLog(t *testing.T) { { name: "no trace", resource: testResource(), + scope: testScope(), log: testLogRecordWithoutTrace(), config: &Config{ LogGroupName: "tLogGroup", @@ -98,7 +120,7 @@ func TestLogToCWLog(t *testing.T) { GeneratedTime: time.Now(), InputLogEvent: &cloudwatchlogs.InputLogEvent{ Timestamp: aws.Int64(1609719139), - Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"attributes":{"key1":1,"key2":"attr2"},"resource":{"host":"abc123","node":5}}`), + Message: aws.String(`{"body":"hello world","severity_number":5,"severity_text":"debug","dropped_attributes_count":4,"attributes":{"key1":1,"key2":"attr2"},"scope":{"name":"test-scope","version":"1.0.0","attributes":{"scope-attr":"value"}},"resource":{"host":"abc123","node":5}}`), }, StreamKey: cwlogs.StreamKey{ LogGroupName: "tLogGroup", @@ -109,6 +131,7 @@ func TestLogToCWLog(t *testing.T) { { name: "raw", resource: testResource(), + scope: testScope(), log: testLogRecordWithoutTrace(), config: &Config{ LogGroupName: "tLogGroup", @@ -130,6 +153,7 @@ func TestLogToCWLog(t *testing.T) { { name: "raw emf v1", resource: testResource(), + scope: testScope(), log: createPLog(`{"_aws":{"Timestamp":1574109732004,"LogGroupName":"Foo","CloudWatchMetrics":[{"Namespace":"MyApp","Dimensions":[["Operation"]],"Metrics":[{"Name":"ProcessingLatency","Unit":"Milliseconds","StorageResolution":60}]}]},"Operation":"Aggregator","ProcessingLatency":100}`), config: &Config{ LogGroupName: "tLogGroup", @@ -151,6 +175,7 @@ func TestLogToCWLog(t *testing.T) { { name: "raw emf v1 with log stream", resource: testResource(), + scope: testScope(), log: createPLog(`{"_aws":{"Timestamp":1574109732004,"LogGroupName":"Foo","LogStreamName":"Foo","CloudWatchMetrics":[{"Namespace":"MyApp","Dimensions":[["Operation"]],"Metrics":[{"Name":"ProcessingLatency","Unit":"Milliseconds","StorageResolution":60}]}]},"Operation":"Aggregator","ProcessingLatency":100}`), config: &Config{ LogGroupName: "tLogGroup", @@ -172,6 +197,7 @@ func TestLogToCWLog(t *testing.T) { { name: "raw emf v0", resource: testResource(), + scope: testScope(), log: createPLog(`{"Timestamp":1574109732004,"log_group_name":"Foo","CloudWatchMetrics":[{"Namespace":"MyApp","Dimensions":[["Operation"]],"Metrics":[{"Name":"ProcessingLatency","Unit":"Milliseconds","StorageResolution":60}]}],"Operation":"Aggregator","ProcessingLatency":100}`), config: &Config{ LogGroupName: "tLogGroup", @@ -193,6 +219,7 @@ func TestLogToCWLog(t *testing.T) { { name: "raw emf v0 with log stream", resource: testResource(), + scope: testScope(), log: createPLog(`{"Timestamp":1574109732004,"log_group_name":"Foo","log_stream_name":"Foo","CloudWatchMetrics":[{"Namespace":"MyApp","Dimensions":[["Operation"]],"Metrics":[{"Name":"ProcessingLatency","Unit":"Milliseconds","StorageResolution":60}]}],"Operation":"Aggregator","ProcessingLatency":100}`), config: &Config{ LogGroupName: "tLogGroup", @@ -216,13 +243,13 @@ func TestLogToCWLog(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resourceAttrs := attrsValue(tt.resource.Attributes()) - got, err := logToCWLog(resourceAttrs, tt.log, tt.config) + got, err := logToCWLog(resourceAttrs, tt.scope, tt.log, tt.config) if (err != nil) != tt.wantErr { t.Errorf("logToCWLog() error = %v, wantErr %v", err, tt.wantErr) return } // Do not test generated time since it is time.Now() - assert.Equal(t, tt.want.InputLogEvent, got.InputLogEvent) + assert.Equal(t, *tt.want.InputLogEvent, *got.InputLogEvent) assert.Equal(t, tt.want.LogStreamName, got.LogStreamName) assert.Equal(t, tt.want.LogGroupName, got.LogGroupName) }) @@ -234,8 +261,9 @@ func BenchmarkLogToCWLog(b *testing.B) { resource := testResource() log := testLogRecord() + scope := testScope() for i := 0; i < b.N; i++ { - _, err := logToCWLog(attrsValue(resource.Attributes()), log, &Config{}) + _, err := logToCWLog(attrsValue(resource.Attributes()), scope, log, &Config{}) if err != nil { b.Errorf("logToCWLog() failed %v", err) return @@ -250,6 +278,19 @@ func testResource() pcommon.Resource { return resource } +func testScope() pcommon.InstrumentationScope { + scope := pcommon.NewInstrumentationScope() + scope.SetName("test-scope") + scope.SetVersion("1.0.0") + scope.Attributes().PutStr("scope-attr", "value") + return scope +} + +func emptyScope() pcommon.InstrumentationScope { + scope := pcommon.NewInstrumentationScope() + return scope +} + func testLogRecord() plog.LogRecord { record := plog.NewLogRecord() record.SetSeverityNumber(5)