Skip to content

Commit

Permalink
[exporter/elasticsearch] Support preserving attributes when mapping t…
Browse files Browse the repository at this point in the history
…o ECS (#33670)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
This PR adds support for preserving resource attributes that are valid
ECS fields in addition of mapping them. At the moment the `host.name` is
mapped to the `host.hostname`. Both are valid ECS fields but can differ
in some cases. Since there is no such distinction in SemConv right now,
it does makes sense to preserve both.

refs:
-
https://www.elastic.co/guide/en/ecs/current/ecs-host.html#field-host-name
-
https://www.elastic.co/guide/en/ecs/current/ecs-host.html#field-host-hostname

**Link to tracking Issue:** <Issue number if applicable>

**Testing:** <Describe what testing was performed and which tests were
added.>

Using the testing notes from
#33622.

Stored document:
```json
 {
    "app": {
      "label": {
        "component": "migration-logger"
      }
    },
    "kubernetes": {
      "node": {
        "name": "kind-control-plane"
      },
      "pod": {
        "uid": "0eda57cd-a4ae-4e89-88fa-c771d3bf0c77",
        "name": "daemonset-logs-4sqjq"
      },
      "namespace": "default"
    },
    "agent": {
      "name": "otlp"
    },
    "@timestamp": "2024-06-20T07:27:42.589678923Z",
    "log": {
      "iostream": "stdout",
      "file": {
        "path": "/var/log/pods/default_daemonset-logs-4sqjq_0eda57cd-a4ae-4e89-88fa-c771d3bf0c77/busybox/0.log"
      }
    },
    "service": {
      "name": "migration-logger"
    },
    "k8s": {
      "container": {
        "restart_count": "0",
        "name": "busybox"
      },
      "pod": {
        "start_time": "2024-06-20T07:27:21Z"
      },
      "daemonset": {
        "name": "daemonset-logs"
      }
    },
    "host": {
      "hostname": "daemonset-opentelemetry-collector-agent-l6pzp",
      "os": {
        "type": "linux",
        "platform": "linux"
      },
      "name": "daemonset-opentelemetry-collector-agent-l6pzp"
    },
    "time": "2024-06-20T07:27:42.589678923Z",
    "message": "otel logs at 07:27:42",
    "logtag": "F"
  }
```

**Documentation:** <Describe the documentation added.> ~

/cc @lahsivjar @andrzej-stencel @carsonip

---------

Signed-off-by: ChrsMark <chrismarkou92@gmail.com>
  • Loading branch information
ChrsMark authored Jun 21, 2024
1 parent 3e275ab commit 86278d9
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 6 deletions.
27 changes: 27 additions & 0 deletions .chloggen/es_preserve_host_name.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: elasticsearchexporter

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Preserve `host.name` resource attribute in ECS mode

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [33670]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [user]
18 changes: 14 additions & 4 deletions exporter/elasticsearchexporter/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@ var resourceAttrsConversionMap = map[string]string{
"k8s.pod.uid": "kubernetes.pod.uid",
}

// resourceAttrsToPreserve contains conventions that should be preserved in ECS mode.
// This can happen when an attribute needs to be mapped to an ECS equivalent but
// at the same time be preserved to its original form.
var resourceAttrsToPreserve = map[string]bool{
semconv.AttributeHostName: true,
}

type mappingModel interface {
encodeLog(pcommon.Resource, plog.LogRecord, pcommon.InstrumentationScope) ([]byte, error)
encodeSpan(pcommon.Resource, ptrace.Span, pcommon.InstrumentationScope) ([]byte, error)
Expand Down Expand Up @@ -117,13 +124,13 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo
var document objmodel.Document

// First, try to map resource-level attributes to ECS fields.
encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap)
encodeLogAttributesECSMode(&document, resource.Attributes(), resourceAttrsConversionMap, resourceAttrsToPreserve)

// Then, try to map scope-level attributes to ECS fields.
scopeAttrsConversionMap := map[string]string{
// None at the moment
}
encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap)
encodeLogAttributesECSMode(&document, scope.Attributes(), scopeAttrsConversionMap, resourceAttrsToPreserve)

// Finally, try to map record-level attributes to ECS fields.
recordAttrsConversionMap := map[string]string{
Expand All @@ -133,7 +140,7 @@ func (m *encodeModel) encodeLogECSMode(resource pcommon.Resource, record plog.Lo
semconv.AttributeExceptionType: "error.type",
semconv.AttributeExceptionEscaped: "event.error.exception.handled",
}
encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap)
encodeLogAttributesECSMode(&document, record.Attributes(), recordAttrsConversionMap, resourceAttrsToPreserve)

// Handle special cases.
encodeLogAgentNameECSMode(&document, resource)
Expand Down Expand Up @@ -230,7 +237,7 @@ func scopeToAttributes(scope pcommon.InstrumentationScope) pcommon.Map {
return attrs
}

func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string) {
func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map, conversionMap map[string]string, preserveMap map[string]bool) {
if len(conversionMap) == 0 {
// No conversions to be done; add all attributes at top level of
// document.
Expand All @@ -247,6 +254,9 @@ func encodeLogAttributesECSMode(document *objmodel.Document, attrs pcommon.Map,
}

document.AddAttribute(ecsKey, v)
if preserve := preserveMap[k]; preserve {
document.AddAttribute(k, v)
}
return true
}

Expand Down
25 changes: 23 additions & 2 deletions exporter/elasticsearchexporter/model_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func TestEncodeLogECSModeDuplication(t *testing.T) {
})
require.NoError(t, err)

want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","name":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
require.NoError(t, err)

resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags)
Expand Down Expand Up @@ -336,6 +336,7 @@ func TestEncodeLogECSMode(t *testing.T) {
"container.image.name": "my-app",
"container.runtime": "docker",
"host.hostname": "i-103de39e0a.gke.us-west-1b.cloud.google.com",
"host.name": "i-103de39e0a.gke.us-west-1b.cloud.google.com",
"host.id": "i-103de39e0a",
"host.type": "t2.medium",
"host.architecture": "x86_64",
Expand Down Expand Up @@ -671,6 +672,7 @@ func TestMapLogAttributesToECS(t *testing.T) {
tests := map[string]struct {
attrs func() pcommon.Map
conversionMap map[string]string
preserveMap map[string]bool
expectedDoc func() objmodel.Document
}{
"no_attrs": {
Expand Down Expand Up @@ -775,12 +777,31 @@ func TestMapLogAttributesToECS(t *testing.T) {
return d
},
},
"preserve_map": {
attrs: func() pcommon.Map {
m := pcommon.NewMap()
m.PutStr("foo.bar", "baz")
return m
},
conversionMap: map[string]string{
"foo.bar": "bar.qux",
"qux": "foo",
}, preserveMap: map[string]bool{
"foo.bar": true,
},
expectedDoc: func() objmodel.Document {
d := objmodel.Document{}
d.AddString("bar.qux", "baz")
d.AddString("foo.bar", "baz")
return d
},
},
}

for name, test := range tests {
t.Run(name, func(t *testing.T) {
var doc objmodel.Document
encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap)
encodeLogAttributesECSMode(&doc, test.attrs(), test.conversionMap, test.preserveMap)

doc.Sort()
expectedDoc := test.expectedDoc()
Expand Down

0 comments on commit 86278d9

Please sign in to comment.