From 1030c3254c15d47e25b033a19d3c57a0fbd6143f Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Thu, 16 Jan 2025 13:02:25 +0800 Subject: [PATCH 1/2] elasticsearchexporter: remove dedup config This config setting has been deprecated for 6 months and can be safely removed now. --- ...elasticsearchexporter-rm-dedup-config.yaml | 27 ++++++++++++++++ exporter/elasticsearchexporter/README.md | 3 -- exporter/elasticsearchexporter/config.go | 10 ------ .../elasticsearchexporter/exporter_test.go | 2 +- .../elasticsearchexporter/factory_test.go | 32 ------------------- 5 files changed, 28 insertions(+), 46 deletions(-) create mode 100644 .chloggen/elasticsearchexporter-rm-dedup-config.yaml diff --git a/.chloggen/elasticsearchexporter-rm-dedup-config.yaml b/.chloggen/elasticsearchexporter-rm-dedup-config.yaml new file mode 100644 index 000000000000..6f9eabb9f020 --- /dev/null +++ b/.chloggen/elasticsearchexporter-rm-dedup-config.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove `dedup` config setting that was deprecated in v0.104.0 + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33772] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 5697db15c4a2..57646629cfc2 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -166,9 +166,6 @@ behaviours, which may be configured through the following settings: It works only for logs where the log record body is a map. Each LogRecord body is serialized to JSON as-is and becomes a separate document for ingestion. If the log record body is not a map, the exporter will log a warning and drop the log record. - - `dedup` (DEPRECATED). This configuration is deprecated and non-operational, - and will be removed in the future. Object keys are always deduplicated to - avoid Elasticsearch rejecting documents. - `dedot` (default=true; DEPRECATED, in future dedotting will always be enabled for ECS mode, and never for other modes): When enabled attributes with `.` will be split into proper json objects. diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 12dc5651204a..40acdaf99497 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -183,13 +183,6 @@ type MappingsSettings struct { // Mode configures the field mappings. Mode string `mapstructure:"mode"` - // Dedup is non-operational, and will be removed in the future. - // - // Deprecated: [v0.104.0] deduplication is always enabled, and cannot be - // disabled. Disabling deduplication is not meaningful, as Elasticsearch - // will always reject documents with duplicate JSON object keys. - Dedup *bool `mapstructure:"dedup,omitempty"` - // Deprecated: [v0.104.0] dedotting will always be applied for ECS mode // in future, and never for other modes. Elasticsearch's "dot_expander" // Ingest processor may be used as an alternative for non-ECS modes. @@ -363,9 +356,6 @@ func (cfg *Config) MappingMode() MappingMode { } func handleDeprecatedConfig(cfg *Config, logger *zap.Logger) { - if cfg.Mapping.Dedup != nil { - logger.Warn("dedup is deprecated, and is always enabled") - } if cfg.Mapping.Dedot && cfg.MappingMode() != MappingECS || !cfg.Mapping.Dedot && cfg.MappingMode() == MappingECS { logger.Warn("dedot has been deprecated: in the future, dedotting will always be performed in ECS mode only") } diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 6125988ea463..3b0490b15172 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -240,7 +240,7 @@ func TestExporterLogs(t *testing.T) { exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { cfg.Mapping.Mode = "raw" - // dedup is the default + // deduplication is always performed - there is no configuration that controls it }) logs := newLogsWithAttributes( // Scope collides with the top-level "Scope" field, diff --git a/exporter/elasticsearchexporter/factory_test.go b/exporter/elasticsearchexporter/factory_test.go index 80f294f37705..7a36b87658b8 100644 --- a/exporter/elasticsearchexporter/factory_test.go +++ b/exporter/elasticsearchexporter/factory_test.go @@ -61,38 +61,6 @@ func TestFactory_CreateTraces(t *testing.T) { require.NoError(t, exporter.Shutdown(context.Background())) } -func TestFactory_DedupDeprecated(t *testing.T) { - factory := NewFactory() - cfg := withDefaultConfig(func(cfg *Config) { - dedup := false - cfg.Endpoint = "http://testing.invalid:9200" - cfg.Mapping.Dedup = &dedup - cfg.Mapping.Dedot = false // avoid dedot warnings - }) - - loggerCore, logObserver := observer.New(zap.WarnLevel) - set := exportertest.NewNopSettings() - set.Logger = zap.New(loggerCore) - - logsExporter, err := factory.CreateLogs(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, logsExporter.Shutdown(context.Background())) - - tracesExporter, err := factory.CreateTraces(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, tracesExporter.Shutdown(context.Background())) - - metricsExporter, err := factory.CreateMetrics(context.Background(), set, cfg) - require.NoError(t, err) - require.NoError(t, metricsExporter.Shutdown(context.Background())) - - records := logObserver.AllUntimed() - assert.Len(t, records, 3) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[0].Message) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[1].Message) - assert.Equal(t, "dedup is deprecated, and is always enabled", records[2].Message) -} - func TestFactory_DedotDeprecated(t *testing.T) { loggerCore, logObserver := observer.New(zap.WarnLevel) set := exportertest.NewNopSettings() From 895f79558ab5786df45821583218b124f80a33a4 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Thu, 16 Jan 2025 13:26:58 +0800 Subject: [PATCH 2/2] Use the correct issue number --- .chloggen/elasticsearchexporter-rm-dedup-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/elasticsearchexporter-rm-dedup-config.yaml b/.chloggen/elasticsearchexporter-rm-dedup-config.yaml index 6f9eabb9f020..7b3ab00d9c53 100644 --- a/.chloggen/elasticsearchexporter-rm-dedup-config.yaml +++ b/.chloggen/elasticsearchexporter-rm-dedup-config.yaml @@ -10,7 +10,7 @@ component: elasticsearchexporter note: Remove `dedup` config setting that was deprecated in v0.104.0 # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. -issues: [33772] +issues: [33773] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document.