From ca2ca14d94eb84d63c72abdea91c1e9faf767883 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sat, 2 Dec 2023 22:01:35 +0800 Subject: [PATCH 01/10] Logstash format compatibility Signed-off-by: Jared Tan --- ...exporter_support_logstash_index_format.yml | 27 ++++++++++++++ exporter/elasticsearchexporter/README.md | 9 +++++ exporter/elasticsearchexporter/config.go | 16 ++++++-- exporter/elasticsearchexporter/config_test.go | 36 ++++++++++++++++++ exporter/elasticsearchexporter/factory.go | 25 +++++++++---- exporter/elasticsearchexporter/go.mod | 2 + exporter/elasticsearchexporter/go.sum | 5 +++ .../elasticsearchexporter/logs_exporter.go | 25 +++++++++---- .../logs_exporter_test.go | 37 +++++++++++++++++++ .../testdata/config.yaml | 4 ++ .../elasticsearchexporter/trace_exporter.go | 23 ++++++++---- .../traces_exporter_test.go | 36 ++++++++++++++++++ exporter/elasticsearchexporter/util.go | 24 ++++++++++++ exporter/elasticsearchexporter/utils_test.go | 26 +++++++++++++ 14 files changed, 269 insertions(+), 26 deletions(-) create mode 100755 .chloggen/elasticsearchexporter_support_logstash_index_format.yml create mode 100644 exporter/elasticsearchexporter/util.go diff --git a/.chloggen/elasticsearchexporter_support_logstash_index_format.yml b/.chloggen/elasticsearchexporter_support_logstash_index_format.yml new file mode 100755 index 000000000000..dedf91eb324d --- /dev/null +++ b/.chloggen/elasticsearchexporter_support_logstash_index_format.yml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Logstash format compatibility. Traces or Logs data can be written into an index in logstash format. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [29624] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 1a5d2e0f02c8..b1aa878cbe4f 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -44,6 +44,15 @@ This exporter supports sending OpenTelemetry logs to [Elasticsearch](https://www takes resource or span attribute named `elasticsearch.index.prefix` and `elasticsearch.index.suffix` resulting dynamically prefixed / suffixed indexing based on `traces_index`. (priority: resource attribute > span attribute) - `enabled`(default=false): Enable/Disable dynamic index for trace spans +- `logstash_format` (optional): Logstash format compatibility. Traces or Logs data can be written into an index in logstash format. + **Note**: In order to reduce complexity, `traces/logs_index`, `traces/logs_dynamic_index`, and `logstash_format` are not compatible and do not support mixing. + There will always be only one in effect. + - `enabled`(default=false): Enable/Disable Logstash format compatibility. + - `prefix`(default=logstash): When `logstash_format.enabled` is enabled, the index name is composed using a prefix and the date, + e.g: If logstash_prefix is equals to 'mydata' your index will become 'mydata-YYYY.MM.DD'. + The last string appended belongs to the date when the data is being generated. + - `prefix_separator`(default=`-`): Set a separator between logstash_prefix and date. + - `date_format`(default=`%Y.%m.%d`): Time format (based on strftime) to generate the second part of the Index name. - `pipeline` (optional): Optional [Ingest Node](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html) pipeline ID used for processing documents published by the exporter. - `flush`: Event bulk buffer flush settings diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 460d605fe92c..e9f7c104dd2e 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -57,10 +57,18 @@ type Config struct { Pipeline string `mapstructure:"pipeline"` HTTPClientSettings `mapstructure:",squash"` - Discovery DiscoverySettings `mapstructure:"discover"` - Retry RetrySettings `mapstructure:"retry"` - Flush FlushSettings `mapstructure:"flush"` - Mapping MappingsSettings `mapstructure:"mapping"` + Discovery DiscoverySettings `mapstructure:"discover"` + Retry RetrySettings `mapstructure:"retry"` + Flush FlushSettings `mapstructure:"flush"` + Mapping MappingsSettings `mapstructure:"mapping"` + LogstashFormat LogstashFormatSettings `mapstructure:"logstash_format"` +} + +type LogstashFormatSettings struct { + Enabled bool `mapstructure:"enabled"` + Prefix string `mapstructure:"prefix"` + PrefixSeparator string `mapstructure:"prefix_separator"` + DateFormat string `mapstructure:"date_format"` } type DynamicIndexSetting struct { diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index bfe9e638e8e6..1343f24bae49 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -67,6 +67,15 @@ func TestLoad_DeprecatedIndexConfigOption(t *testing.T) { Dedup: true, Dedot: true, }, + LogstashFormat: LogstashFormatSettings{ + Enabled: false, + Prefix: "logstash", + PrefixSeparator: "-", + DateFormat: "%Y.%m.%d", + TimeKey: "@timestamp", + TimeKeyFormat: "%dT%H:%M:%S", + TimeKeyNanos: false, + }, }) } @@ -76,6 +85,10 @@ func TestLoadConfig(t *testing.T) { defaultCfg := createDefaultConfig() defaultCfg.(*Config).Endpoints = []string{"https://elastic.example.com:9200"} + defaultLogstashFormatCfg := createDefaultConfig() + defaultLogstashFormatCfg.(*Config).Endpoints = []string{"https://localhost:9200"} + defaultLogstashFormatCfg.(*Config).LogstashFormat.Enabled = true + tests := []struct { configFile string id component.ID @@ -129,6 +142,15 @@ func TestLoadConfig(t *testing.T) { Dedup: true, Dedot: true, }, + LogstashFormat: LogstashFormatSettings{ + Enabled: false, + Prefix: "logstash", + PrefixSeparator: "-", + DateFormat: "%Y.%m.%d", + TimeKey: "@timestamp", + TimeKeyFormat: "%dT%H:%M:%S", + TimeKeyNanos: false, + }, }, }, { @@ -174,8 +196,22 @@ func TestLoadConfig(t *testing.T) { Dedup: true, Dedot: true, }, + LogstashFormat: LogstashFormatSettings{ + Enabled: false, + Prefix: "logstash", + PrefixSeparator: "-", + DateFormat: "%Y.%m.%d", + TimeKey: "@timestamp", + TimeKeyFormat: "%dT%H:%M:%S", + TimeKeyNanos: false, + }, }, }, + { + id: component.NewIDWithName(metadata.Type, "logstash_format"), + configFile: "config.yaml", + expected: defaultLogstashFormatCfg, + }, } for _, tt := range tests { diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 1f951700d51b..39f226369168 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -55,6 +55,15 @@ func createDefaultConfig() component.Config { Dedup: true, Dedot: true, }, + LogstashFormat: LogstashFormatSettings{ + Enabled: false, + Prefix: "logstash", + PrefixSeparator: "-", + DateFormat: "%Y.%m.%d", + TimeKey: "@timestamp", + TimeKeyFormat: "%dT%H:%M:%S", + TimeKeyNanos: false, + }, } } @@ -71,17 +80,17 @@ func createLogsExporter( set.Logger.Warn("index option are deprecated and replaced with logs_index and traces_index.") } - exporter, err := newLogsExporter(set.Logger, cf) + logsExporter, err := newLogsExporter(set.Logger, cf) if err != nil { - return nil, fmt.Errorf("cannot configure Elasticsearch logs exporter: %w", err) + return nil, fmt.Errorf("cannot configure Elasticsearch logs logsExporter: %w", err) } return exporterhelper.NewLogsExporter( ctx, set, cfg, - exporter.pushLogsData, - exporterhelper.WithShutdown(exporter.Shutdown), + logsExporter.pushLogsData, + exporterhelper.WithShutdown(logsExporter.Shutdown), exporterhelper.WithQueue(cf.QueueSettings), ) } @@ -91,15 +100,15 @@ func createTracesExporter(ctx context.Context, cfg component.Config) (exporter.Traces, error) { cf := cfg.(*Config) - exporter, err := newTracesExporter(set.Logger, cf) + tracesExporter, err := newTracesExporter(set.Logger, cf) if err != nil { - return nil, fmt.Errorf("cannot configure Elasticsearch traces exporter: %w", err) + return nil, fmt.Errorf("cannot configure Elasticsearch traces tracesExporter: %w", err) } return exporterhelper.NewTracesExporter( ctx, set, cfg, - exporter.pushTraceData, - exporterhelper.WithShutdown(exporter.Shutdown), + tracesExporter.pushTraceData, + exporterhelper.WithShutdown(tracesExporter.Shutdown), exporterhelper.WithQueue(cf.QueueSettings)) } diff --git a/exporter/elasticsearchexporter/go.mod b/exporter/elasticsearchexporter/go.mod index 03d222a86dac..041602ff05cf 100644 --- a/exporter/elasticsearchexporter/go.mod +++ b/exporter/elasticsearchexporter/go.mod @@ -29,11 +29,13 @@ require ( github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.0.1 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector v0.90.1 // indirect diff --git a/exporter/elasticsearchexporter/go.sum b/exporter/elasticsearchexporter/go.sum index 4bef43e71510..cb38f865372d 100644 --- a/exporter/elasticsearchexporter/go.sum +++ b/exporter/elasticsearchexporter/go.sum @@ -67,6 +67,9 @@ github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -79,6 +82,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= diff --git a/exporter/elasticsearchexporter/logs_exporter.go b/exporter/elasticsearchexporter/logs_exporter.go index 7511988bc2ba..7cc3e4c86dc7 100644 --- a/exporter/elasticsearchexporter/logs_exporter.go +++ b/exporter/elasticsearchexporter/logs_exporter.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "time" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -18,9 +19,12 @@ import ( type elasticsearchLogsExporter struct { logger *zap.Logger - index string - dynamicIndex bool - maxAttempts int + // if logstash format enabled, this index will ignore and won't be used. + index string + // if logstash format enabled, exporter will use `logstashIndex` as final index write to. + logstashFormat LogstashFormatSettings + dynamicIndex bool + maxAttempts int client *esClientCurrent bulkIndexer esBulkIndexerCurrent @@ -62,10 +66,11 @@ func newLogsExporter(logger *zap.Logger, cfg *Config) (*elasticsearchLogsExporte client: client, bulkIndexer: bulkIndexer, - index: indexStr, - dynamicIndex: cfg.LogsDynamicIndex.Enabled, - maxAttempts: maxAttempts, - model: model, + index: indexStr, + dynamicIndex: cfg.LogsDynamicIndex.Enabled, + maxAttempts: maxAttempts, + model: model, + logstashFormat: cfg.LogstashFormat, } return esLogsExp, nil } @@ -107,6 +112,12 @@ func (e *elasticsearchLogsExporter) pushLogRecord(ctx context.Context, resource suffix := getFromBothResourceAndAttribute(indexSuffix, resource, record) fIndex = fmt.Sprintf("%s%s%s", prefix, fIndex, suffix) + } else { + generatedIndex, err := generateIndex(fIndex, &e.logstashFormat, time.Now()) + if err != nil { + return err + } + fIndex = generatedIndex } document, err := e.model.encodeLog(resource, record, scope) diff --git a/exporter/elasticsearchexporter/logs_exporter_test.go b/exporter/elasticsearchexporter/logs_exporter_test.go index 48fa8a06cc0b..810fa8e4c489 100644 --- a/exporter/elasticsearchexporter/logs_exporter_test.go +++ b/exporter/elasticsearchexporter/logs_exporter_test.go @@ -210,6 +210,43 @@ func TestExporter_PushEvent(t *testing.T) { rec.WaitItems(1) }) + t.Run("publish with logstash format index", func(t *testing.T) { + + defaultCfg := createDefaultConfig().(*Config) + defaultCfg.LogstashFormat.Enabled = true + testTime := time.Date(2023, 12, 2, 10, 10, 10, 1, time.UTC) + expectedIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + assert.Nil(t, err) + assert.Equal(t, expectedIndex, "logstash-2023.12.02") + + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + + data, err := docs[0].Action.MarshalJSON() + assert.Nil(t, err) + + jsonVal := map[string]any{} + err = json.Unmarshal(data, &jsonVal) + assert.Nil(t, err) + + create := jsonVal["create"].(map[string]any) + + assert.Equal(t, expectedIndex, create["_index"].(string)) + + return itemsAllOK(docs) + }) + + exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { + cfg.LogsIndex = "not-used-index" + cfg.LogstashFormat.Enabled = true + }) + + mustSendLogsWithAttributes(t, exporter, nil, nil) + + rec.WaitItems(1) + }) + t.Run("retry http request", func(t *testing.T) { failures := 0 rec := newBulkRecorder() diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index 60ba25ebb476..792640222adb 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -40,3 +40,7 @@ elasticsearch/log: max_requests: 5 sending_queue: enabled: true +elasticsearch/logstash_format: + endpoints: [http://localhost:9200] + logstash_format: + enabled: true diff --git a/exporter/elasticsearchexporter/trace_exporter.go b/exporter/elasticsearchexporter/trace_exporter.go index ef421951ed4c..8c570b97033f 100644 --- a/exporter/elasticsearchexporter/trace_exporter.go +++ b/exporter/elasticsearchexporter/trace_exporter.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "time" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" @@ -18,9 +19,10 @@ import ( type elasticsearchTracesExporter struct { logger *zap.Logger - index string - dynamicIndex bool - maxAttempts int + index string + logstashFormat LogstashFormatSettings + dynamicIndex bool + maxAttempts int client *esClientCurrent bulkIndexer esBulkIndexerCurrent @@ -54,10 +56,11 @@ func newTracesExporter(logger *zap.Logger, cfg *Config) (*elasticsearchTracesExp client: client, bulkIndexer: bulkIndexer, - index: cfg.TracesIndex, - dynamicIndex: cfg.TracesDynamicIndex.Enabled, - maxAttempts: maxAttempts, - model: model, + index: cfg.TracesIndex, + dynamicIndex: cfg.TracesDynamicIndex.Enabled, + maxAttempts: maxAttempts, + model: model, + logstashFormat: cfg.LogstashFormat, }, nil } @@ -99,6 +102,12 @@ func (e *elasticsearchTracesExporter) pushTraceRecord(ctx context.Context, resou suffix := getFromBothResourceAndAttribute(indexSuffix, resource, span) fIndex = fmt.Sprintf("%s%s%s", prefix, fIndex, suffix) + } else { + generatedIndex, err := generateIndex(fIndex, &e.logstashFormat, time.Now()) + if err != nil { + return err + } + fIndex = generatedIndex } document, err := e.model.encodeSpan(resource, span, scope) diff --git a/exporter/elasticsearchexporter/traces_exporter_test.go b/exporter/elasticsearchexporter/traces_exporter_test.go index 7c40be6b4d61..224904bdb316 100644 --- a/exporter/elasticsearchexporter/traces_exporter_test.go +++ b/exporter/elasticsearchexporter/traces_exporter_test.go @@ -200,6 +200,42 @@ func TestExporter_PushTraceRecord(t *testing.T) { rec.WaitItems(1) }) + t.Run("publish with logstash format index", func(t *testing.T) { + defaultCfg := createDefaultConfig().(*Config) + defaultCfg.LogstashFormat.Enabled = true + testTime := time.Date(2023, 12, 2, 10, 10, 10, 1, time.UTC) + expectedIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + assert.Nil(t, err) + assert.Equal(t, expectedIndex, "logstash-2023.12.02") + + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + + data, err := docs[0].Action.MarshalJSON() + assert.Nil(t, err) + + jsonVal := map[string]any{} + err = json.Unmarshal(data, &jsonVal) + assert.Nil(t, err) + + create := jsonVal["create"].(map[string]any) + + assert.Equal(t, expectedIndex, create["_index"].(string)) + + return itemsAllOK(docs) + }) + + exporter := newTestTracesExporter(t, server.URL, func(cfg *Config) { + cfg.LogsIndex = "not-used-index" + cfg.LogstashFormat.Enabled = true + }) + + mustSendTracesWithAttributes(t, exporter, nil, nil) + + rec.WaitItems(1) + }) + t.Run("retry http request", func(t *testing.T) { failures := 0 rec := newBulkRecorder() diff --git a/exporter/elasticsearchexporter/util.go b/exporter/elasticsearchexporter/util.go new file mode 100644 index 000000000000..8eec3c8f9097 --- /dev/null +++ b/exporter/elasticsearchexporter/util.go @@ -0,0 +1,24 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package elasticsearchexporter + +import ( + "bytes" + "fmt" + "github.com/lestrrat-go/strftime" + "time" +) + +func generateIndex(index string, conf *LogstashFormatSettings, t time.Time) (string, error) { + if conf.Enabled { + partIndex := fmt.Sprintf("%s%s", conf.Prefix, conf.PrefixSeparator) + var buf bytes.Buffer + p, err := strftime.New(fmt.Sprintf("%s%s", partIndex, conf.DateFormat)) + if err = p.Format(&buf, t); err != nil { + return partIndex, err + } + index = buf.String() + } + return index, nil +} diff --git a/exporter/elasticsearchexporter/utils_test.go b/exporter/elasticsearchexporter/utils_test.go index 3309d9fb9a26..815769fc1451 100644 --- a/exporter/elasticsearchexporter/utils_test.go +++ b/exporter/elasticsearchexporter/utils_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "strings" @@ -251,3 +252,28 @@ func fillResourceAttributeMap(attrs pcommon.Map, mp map[string]string) { attrs.PutStr(k, v) } } + +func TestGetSuffixTime(t *testing.T) { + defaultCfg := createDefaultConfig().(*Config) + defaultCfg.LogstashFormat.Enabled = true + testTime := time.Date(2023, 12, 2, 10, 10, 10, 1, time.UTC) + index, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + assert.Nil(t, err) + assert.Equal(t, index, "logstash-2023.12.02") + + defaultCfg.LogstashFormat.Prefix = "otel-logs" + defaultCfg.LogstashFormat.PrefixSeparator = "." + otelLogsIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + assert.Nil(t, err) + assert.Equal(t, otelLogsIndex, "otel-logs.2023.12.02") + + defaultCfg.LogstashFormat.DateFormat = "%Y-%m-%d" + newOtelLogsIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + assert.Nil(t, err) + assert.Equal(t, newOtelLogsIndex, "otel-logs.2023-12-02") + + defaultCfg.LogstashFormat.DateFormat = "%d/%m/%Y" + newOtelLogsIndexWithSpecDataFormat, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + assert.Nil(t, err) + assert.Equal(t, newOtelLogsIndexWithSpecDataFormat, "otel-logs.02/12/2023") +} From 8d0534affb5229585ff5d244a835fea25930f7d7 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sat, 2 Dec 2023 22:10:05 +0800 Subject: [PATCH 02/10] rename chlog file Signed-off-by: Jared Tan --- ... elasticsearchexporter_support_logstash_index_format.yaml} | 0 cmd/configschema/go.mod | 1 + cmd/configschema/go.sum | 4 ++++ cmd/otelcontribcol/go.mod | 1 + cmd/otelcontribcol/go.sum | 4 ++++ go.mod | 1 + go.sum | 4 ++++ 7 files changed, 15 insertions(+) rename .chloggen/{elasticsearchexporter_support_logstash_index_format.yml => elasticsearchexporter_support_logstash_index_format.yaml} (100%) diff --git a/.chloggen/elasticsearchexporter_support_logstash_index_format.yml b/.chloggen/elasticsearchexporter_support_logstash_index_format.yaml similarity index 100% rename from .chloggen/elasticsearchexporter_support_logstash_index_format.yml rename to .chloggen/elasticsearchexporter_support_logstash_index_format.yaml diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index ddef37c62cd1..012cec5af72b 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -436,6 +436,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect github.com/leoluk/perflib_exporter v0.2.1 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lib/pq v1.10.9 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index 24ec4b66d007..b59ce0f41d01 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -1075,6 +1075,10 @@ github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b h1:11UHH39 github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leoluk/perflib_exporter v0.2.1 h1:/3/ut1k/jFt5p4ypjLZKDHDqlXAK6ERZPVWtwdI389I= github.com/leoluk/perflib_exporter v0.2.1/go.mod h1:MinSWm88jguXFFrGsP56PtleUb4Qtm4tNRH/wXNXRTI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 8ef3b4a62eb6..f3a86d29811a 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -484,6 +484,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/leoluk/perflib_exporter v0.2.1 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lib/pq v1.10.9 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index 7800d3ea5a24..359bcb56d0f4 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -1071,6 +1071,10 @@ github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRC github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leoluk/perflib_exporter v0.2.1 h1:/3/ut1k/jFt5p4ypjLZKDHDqlXAK6ERZPVWtwdI389I= github.com/leoluk/perflib_exporter v0.2.1/go.mod h1:MinSWm88jguXFFrGsP56PtleUb4Qtm4tNRH/wXNXRTI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= diff --git a/go.mod b/go.mod index 36ec9ae162af..2ceb278a74ce 100644 --- a/go.mod +++ b/go.mod @@ -464,6 +464,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/leoluk/perflib_exporter v0.2.1 // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lib/pq v1.10.9 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect diff --git a/go.sum b/go.sum index 3add4b6f679e..54e867742c61 100644 --- a/go.sum +++ b/go.sum @@ -1080,6 +1080,10 @@ github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRC github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/leoluk/perflib_exporter v0.2.1 h1:/3/ut1k/jFt5p4ypjLZKDHDqlXAK6ERZPVWtwdI389I= github.com/leoluk/perflib_exporter v0.2.1/go.mod h1:MinSWm88jguXFFrGsP56PtleUb4Qtm4tNRH/wXNXRTI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= From d9fa603b31319a2b258ad081261af8ec157c5a3e Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sun, 3 Dec 2023 13:32:18 +0800 Subject: [PATCH 03/10] fix ci Signed-off-by: Jared Tan --- exporter/elasticsearchexporter/factory.go | 3 --- exporter/elasticsearchexporter/util.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 39f226369168..50a70e8917f5 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -60,9 +60,6 @@ func createDefaultConfig() component.Config { Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", - TimeKey: "@timestamp", - TimeKeyFormat: "%dT%H:%M:%S", - TimeKeyNanos: false, }, } } diff --git a/exporter/elasticsearchexporter/util.go b/exporter/elasticsearchexporter/util.go index 8eec3c8f9097..acd784a6214c 100644 --- a/exporter/elasticsearchexporter/util.go +++ b/exporter/elasticsearchexporter/util.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package elasticsearchexporter +package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter" import ( "bytes" From 2054202cef8efdc856271361f7ac8507618a5f7c Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sun, 3 Dec 2023 16:45:55 +0800 Subject: [PATCH 04/10] fix ci Signed-off-by: Jared Tan --- exporter/elasticsearchexporter/config_test.go | 9 --------- exporter/elasticsearchexporter/go.mod | 2 +- exporter/elasticsearchexporter/go.sum | 1 + 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index 1343f24bae49..e8e97023167b 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -72,9 +72,6 @@ func TestLoad_DeprecatedIndexConfigOption(t *testing.T) { Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", - TimeKey: "@timestamp", - TimeKeyFormat: "%dT%H:%M:%S", - TimeKeyNanos: false, }, }) } @@ -147,9 +144,6 @@ func TestLoadConfig(t *testing.T) { Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", - TimeKey: "@timestamp", - TimeKeyFormat: "%dT%H:%M:%S", - TimeKeyNanos: false, }, }, }, @@ -201,9 +195,6 @@ func TestLoadConfig(t *testing.T) { Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", - TimeKey: "@timestamp", - TimeKeyFormat: "%dT%H:%M:%S", - TimeKeyNanos: false, }, }, }, diff --git a/exporter/elasticsearchexporter/go.mod b/exporter/elasticsearchexporter/go.mod index 041602ff05cf..14da574a97db 100644 --- a/exporter/elasticsearchexporter/go.mod +++ b/exporter/elasticsearchexporter/go.mod @@ -6,6 +6,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-structform v0.0.10 + github.com/lestrrat-go/strftime v1.0.6 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.90.1 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.90.1 github.com/stretchr/testify v1.8.4 @@ -29,7 +30,6 @@ require ( github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.0.1 // indirect - github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect diff --git a/exporter/elasticsearchexporter/go.sum b/exporter/elasticsearchexporter/go.sum index cb38f865372d..ac27639ed86d 100644 --- a/exporter/elasticsearchexporter/go.sum +++ b/exporter/elasticsearchexporter/go.sum @@ -67,6 +67,7 @@ github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= From 598b9f8e958d5717b9ef4d75139f937e6fd0f040 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sun, 3 Dec 2023 18:32:28 +0800 Subject: [PATCH 05/10] fix lint Signed-off-by: Jared Tan --- exporter/elasticsearchexporter/config_test.go | 2 +- exporter/elasticsearchexporter/util.go | 6 +++++- exporter/elasticsearchexporter/utils_test.go | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index e8e97023167b..b4bef4b40f76 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -83,7 +83,7 @@ func TestLoadConfig(t *testing.T) { defaultCfg.(*Config).Endpoints = []string{"https://elastic.example.com:9200"} defaultLogstashFormatCfg := createDefaultConfig() - defaultLogstashFormatCfg.(*Config).Endpoints = []string{"https://localhost:9200"} + defaultLogstashFormatCfg.(*Config).Endpoints = []string{"http://localhost:9200"} defaultLogstashFormatCfg.(*Config).LogstashFormat.Enabled = true tests := []struct { diff --git a/exporter/elasticsearchexporter/util.go b/exporter/elasticsearchexporter/util.go index acd784a6214c..5d7a85a89eb5 100644 --- a/exporter/elasticsearchexporter/util.go +++ b/exporter/elasticsearchexporter/util.go @@ -6,8 +6,9 @@ package elasticsearchexporter // import "github.com/open-telemetry/opentelemetry import ( "bytes" "fmt" - "github.com/lestrrat-go/strftime" "time" + + "github.com/lestrrat-go/strftime" ) func generateIndex(index string, conf *LogstashFormatSettings, t time.Time) (string, error) { @@ -15,6 +16,9 @@ func generateIndex(index string, conf *LogstashFormatSettings, t time.Time) (str partIndex := fmt.Sprintf("%s%s", conf.Prefix, conf.PrefixSeparator) var buf bytes.Buffer p, err := strftime.New(fmt.Sprintf("%s%s", partIndex, conf.DateFormat)) + if err != nil { + return partIndex, err + } if err = p.Format(&buf, t); err != nil { return partIndex, err } diff --git a/exporter/elasticsearchexporter/utils_test.go b/exporter/elasticsearchexporter/utils_test.go index 815769fc1451..4311f78ac30c 100644 --- a/exporter/elasticsearchexporter/utils_test.go +++ b/exporter/elasticsearchexporter/utils_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "strings" @@ -16,6 +15,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/ptrace" From e99065105cbc4119c10caa37effa924d9530fcf2 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Sun, 3 Dec 2023 19:32:48 +0800 Subject: [PATCH 06/10] fix ut Signed-off-by: Jared Tan --- .../elasticsearchexporter/logs_exporter_test.go | 15 +++++---------- .../elasticsearchexporter/traces_exporter_test.go | 13 +++++-------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/exporter/elasticsearchexporter/logs_exporter_test.go b/exporter/elasticsearchexporter/logs_exporter_test.go index 810fa8e4c489..d703d3d252b6 100644 --- a/exporter/elasticsearchexporter/logs_exporter_test.go +++ b/exporter/elasticsearchexporter/logs_exporter_test.go @@ -10,6 +10,7 @@ import ( "fmt" "net/http" "runtime" + "strings" "sync" "sync/atomic" "testing" @@ -211,14 +212,7 @@ func TestExporter_PushEvent(t *testing.T) { }) t.Run("publish with logstash format index", func(t *testing.T) { - - defaultCfg := createDefaultConfig().(*Config) - defaultCfg.LogstashFormat.Enabled = true - testTime := time.Date(2023, 12, 2, 10, 10, 10, 1, time.UTC) - expectedIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) - assert.Nil(t, err) - assert.Equal(t, expectedIndex, "logstash-2023.12.02") - + var defaultCfg Config rec := newBulkRecorder() server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { rec.Record(docs) @@ -232,14 +226,15 @@ func TestExporter_PushEvent(t *testing.T) { create := jsonVal["create"].(map[string]any) - assert.Equal(t, expectedIndex, create["_index"].(string)) + assert.Equal(t, strings.Contains(create["_index"].(string), defaultCfg.LogstashFormat.Prefix), true) return itemsAllOK(docs) }) exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { - cfg.LogsIndex = "not-used-index" cfg.LogstashFormat.Enabled = true + cfg.LogsIndex = "not-used-index" + defaultCfg = *cfg }) mustSendLogsWithAttributes(t, exporter, nil, nil) diff --git a/exporter/elasticsearchexporter/traces_exporter_test.go b/exporter/elasticsearchexporter/traces_exporter_test.go index 224904bdb316..f5bcfff3ab58 100644 --- a/exporter/elasticsearchexporter/traces_exporter_test.go +++ b/exporter/elasticsearchexporter/traces_exporter_test.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "runtime" + "strings" "sync" "sync/atomic" "testing" @@ -201,12 +202,7 @@ func TestExporter_PushTraceRecord(t *testing.T) { }) t.Run("publish with logstash format index", func(t *testing.T) { - defaultCfg := createDefaultConfig().(*Config) - defaultCfg.LogstashFormat.Enabled = true - testTime := time.Date(2023, 12, 2, 10, 10, 10, 1, time.UTC) - expectedIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) - assert.Nil(t, err) - assert.Equal(t, expectedIndex, "logstash-2023.12.02") + var defaultCfg Config rec := newBulkRecorder() server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { @@ -221,14 +217,15 @@ func TestExporter_PushTraceRecord(t *testing.T) { create := jsonVal["create"].(map[string]any) - assert.Equal(t, expectedIndex, create["_index"].(string)) + assert.Equal(t, strings.Contains(create["_index"].(string), defaultCfg.LogstashFormat.Prefix), true) return itemsAllOK(docs) }) exporter := newTestTracesExporter(t, server.URL, func(cfg *Config) { - cfg.LogsIndex = "not-used-index" cfg.LogstashFormat.Enabled = true + cfg.TracesIndex = "not-used-index" + defaultCfg = *cfg }) mustSendTracesWithAttributes(t, exporter, nil, nil) From dd68013e098dac370ca1feb89776bb2ea715555d Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Tue, 5 Dec 2023 09:25:00 +0800 Subject: [PATCH 07/10] polish logical Signed-off-by: Jared Tan --- exporter/elasticsearchexporter/README.md | 7 +-- exporter/elasticsearchexporter/config.go | 1 - exporter/elasticsearchexporter/config_test.go | 3 -- exporter/elasticsearchexporter/factory.go | 1 - .../elasticsearchexporter/logs_exporter.go | 8 ++-- .../logs_exporter_test.go | 47 ++++++++++++++++++- .../elasticsearchexporter/trace_exporter.go | 8 ++-- .../traces_exporter_test.go | 46 +++++++++++++++++- exporter/elasticsearchexporter/util.go | 4 +- exporter/elasticsearchexporter/utils_test.go | 18 +++---- 10 files changed, 113 insertions(+), 30 deletions(-) diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index b1aa878cbe4f..caed01f44022 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -45,11 +45,8 @@ This exporter supports sending OpenTelemetry logs to [Elasticsearch](https://www resulting dynamically prefixed / suffixed indexing based on `traces_index`. (priority: resource attribute > span attribute) - `enabled`(default=false): Enable/Disable dynamic index for trace spans - `logstash_format` (optional): Logstash format compatibility. Traces or Logs data can be written into an index in logstash format. - **Note**: In order to reduce complexity, `traces/logs_index`, `traces/logs_dynamic_index`, and `logstash_format` are not compatible and do not support mixing. - There will always be only one in effect. - - `enabled`(default=false): Enable/Disable Logstash format compatibility. - - `prefix`(default=logstash): When `logstash_format.enabled` is enabled, the index name is composed using a prefix and the date, - e.g: If logstash_prefix is equals to 'mydata' your index will become 'mydata-YYYY.MM.DD'. + - `enabled`(default=false): Enable/Disable Logstash format compatibility. When `logstash_format.enabled` is `true`, the index name is composed using `traces/logs_index` or `traces/logs_dynamic_index` as prefix and the date, + e.g: If `traces/logs_index` or `traces/logs_dynamic_index` is equals to `otlp-generic-default` your index will become `otlp-generic-default-YYYY.MM.DD`. The last string appended belongs to the date when the data is being generated. - `prefix_separator`(default=`-`): Set a separator between logstash_prefix and date. - `date_format`(default=`%Y.%m.%d`): Time format (based on strftime) to generate the second part of the Index name. diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index e9f7c104dd2e..2240bf3acd2a 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -66,7 +66,6 @@ type Config struct { type LogstashFormatSettings struct { Enabled bool `mapstructure:"enabled"` - Prefix string `mapstructure:"prefix"` PrefixSeparator string `mapstructure:"prefix_separator"` DateFormat string `mapstructure:"date_format"` } diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index b4bef4b40f76..91ccfe638ef8 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -69,7 +69,6 @@ func TestLoad_DeprecatedIndexConfigOption(t *testing.T) { }, LogstashFormat: LogstashFormatSettings{ Enabled: false, - Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", }, @@ -141,7 +140,6 @@ func TestLoadConfig(t *testing.T) { }, LogstashFormat: LogstashFormatSettings{ Enabled: false, - Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", }, @@ -192,7 +190,6 @@ func TestLoadConfig(t *testing.T) { }, LogstashFormat: LogstashFormatSettings{ Enabled: false, - Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", }, diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 50a70e8917f5..45d5bc8d2373 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -57,7 +57,6 @@ func createDefaultConfig() component.Config { }, LogstashFormat: LogstashFormatSettings{ Enabled: false, - Prefix: "logstash", PrefixSeparator: "-", DateFormat: "%Y.%m.%d", }, diff --git a/exporter/elasticsearchexporter/logs_exporter.go b/exporter/elasticsearchexporter/logs_exporter.go index 7cc3e4c86dc7..3fe9c889d564 100644 --- a/exporter/elasticsearchexporter/logs_exporter.go +++ b/exporter/elasticsearchexporter/logs_exporter.go @@ -112,12 +112,14 @@ func (e *elasticsearchLogsExporter) pushLogRecord(ctx context.Context, resource suffix := getFromBothResourceAndAttribute(indexSuffix, resource, record) fIndex = fmt.Sprintf("%s%s%s", prefix, fIndex, suffix) - } else { - generatedIndex, err := generateIndex(fIndex, &e.logstashFormat, time.Now()) + } + + if e.logstashFormat.Enabled { + formattedIndex, err := generateIndexWithLogstashFormat(fIndex, &e.logstashFormat, time.Now()) if err != nil { return err } - fIndex = generatedIndex + fIndex = formattedIndex } document, err := e.model.encodeLog(resource, record, scope) diff --git a/exporter/elasticsearchexporter/logs_exporter_test.go b/exporter/elasticsearchexporter/logs_exporter_test.go index d703d3d252b6..c7ae40c8783c 100644 --- a/exporter/elasticsearchexporter/logs_exporter_test.go +++ b/exporter/elasticsearchexporter/logs_exporter_test.go @@ -211,7 +211,7 @@ func TestExporter_PushEvent(t *testing.T) { rec.WaitItems(1) }) - t.Run("publish with logstash format index", func(t *testing.T) { + t.Run("publish with logstash index format enabled and dynamic index disabled", func(t *testing.T) { var defaultCfg Config rec := newBulkRecorder() server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { @@ -226,7 +226,7 @@ func TestExporter_PushEvent(t *testing.T) { create := jsonVal["create"].(map[string]any) - assert.Equal(t, strings.Contains(create["_index"].(string), defaultCfg.LogstashFormat.Prefix), true) + assert.Equal(t, strings.Contains(create["_index"].(string), defaultCfg.LogsIndex), true) return itemsAllOK(docs) }) @@ -242,6 +242,49 @@ func TestExporter_PushEvent(t *testing.T) { rec.WaitItems(1) }) + t.Run("publish with logstash index format enabled and dynamic index enabled", func(t *testing.T) { + var ( + prefix = "resprefix-" + suffix = "-attrsuffix" + index = "someindex" + ) + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + + data, err := docs[0].Action.MarshalJSON() + assert.Nil(t, err) + + jsonVal := map[string]any{} + err = json.Unmarshal(data, &jsonVal) + assert.Nil(t, err) + + create := jsonVal["create"].(map[string]any) + expected := fmt.Sprintf("%s%s%s", prefix, index, suffix) + + assert.Equal(t, strings.Contains(create["_index"].(string), expected), true) + + return itemsAllOK(docs) + }) + + exporter := newTestLogsExporter(t, server.URL, func(cfg *Config) { + cfg.LogsIndex = index + cfg.LogsDynamicIndex.Enabled = true + cfg.LogstashFormat.Enabled = true + }) + + mustSendLogsWithAttributes(t, exporter, + map[string]string{ + indexPrefix: "attrprefix-", + indexSuffix: suffix, + }, + map[string]string{ + indexPrefix: prefix, + }, + ) + rec.WaitItems(1) + }) + t.Run("retry http request", func(t *testing.T) { failures := 0 rec := newBulkRecorder() diff --git a/exporter/elasticsearchexporter/trace_exporter.go b/exporter/elasticsearchexporter/trace_exporter.go index 8c570b97033f..bfa3c485271f 100644 --- a/exporter/elasticsearchexporter/trace_exporter.go +++ b/exporter/elasticsearchexporter/trace_exporter.go @@ -102,12 +102,14 @@ func (e *elasticsearchTracesExporter) pushTraceRecord(ctx context.Context, resou suffix := getFromBothResourceAndAttribute(indexSuffix, resource, span) fIndex = fmt.Sprintf("%s%s%s", prefix, fIndex, suffix) - } else { - generatedIndex, err := generateIndex(fIndex, &e.logstashFormat, time.Now()) + } + + if e.logstashFormat.Enabled { + formattedIndex, err := generateIndexWithLogstashFormat(fIndex, &e.logstashFormat, time.Now()) if err != nil { return err } - fIndex = generatedIndex + fIndex = formattedIndex } document, err := e.model.encodeSpan(resource, span, scope) diff --git a/exporter/elasticsearchexporter/traces_exporter_test.go b/exporter/elasticsearchexporter/traces_exporter_test.go index f5bcfff3ab58..8cbc5c3e640b 100644 --- a/exporter/elasticsearchexporter/traces_exporter_test.go +++ b/exporter/elasticsearchexporter/traces_exporter_test.go @@ -217,7 +217,7 @@ func TestExporter_PushTraceRecord(t *testing.T) { create := jsonVal["create"].(map[string]any) - assert.Equal(t, strings.Contains(create["_index"].(string), defaultCfg.LogstashFormat.Prefix), true) + assert.Equal(t, strings.Contains(create["_index"].(string), defaultCfg.TracesIndex), true) return itemsAllOK(docs) }) @@ -233,6 +233,50 @@ func TestExporter_PushTraceRecord(t *testing.T) { rec.WaitItems(1) }) + t.Run("publish with logstash format index and dynamic index enabled ", func(t *testing.T) { + var ( + prefix = "resprefix-" + suffix = "-attrsuffix" + index = "someindex" + ) + + rec := newBulkRecorder() + server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) { + rec.Record(docs) + + data, err := docs[0].Action.MarshalJSON() + assert.Nil(t, err) + + jsonVal := map[string]any{} + err = json.Unmarshal(data, &jsonVal) + assert.Nil(t, err) + + create := jsonVal["create"].(map[string]any) + expected := fmt.Sprintf("%s%s%s", prefix, index, suffix) + + assert.Equal(t, strings.Contains(create["_index"].(string), expected), true) + + return itemsAllOK(docs) + }) + + exporter := newTestTracesExporter(t, server.URL, func(cfg *Config) { + cfg.TracesIndex = index + cfg.TracesDynamicIndex.Enabled = true + cfg.LogstashFormat.Enabled = true + }) + + mustSendTracesWithAttributes(t, exporter, + map[string]string{ + indexPrefix: "attrprefix-", + indexSuffix: suffix, + }, + map[string]string{ + indexPrefix: prefix, + }, + ) + rec.WaitItems(1) + }) + t.Run("retry http request", func(t *testing.T) { failures := 0 rec := newBulkRecorder() diff --git a/exporter/elasticsearchexporter/util.go b/exporter/elasticsearchexporter/util.go index 5d7a85a89eb5..e5b398082b3b 100644 --- a/exporter/elasticsearchexporter/util.go +++ b/exporter/elasticsearchexporter/util.go @@ -11,9 +11,9 @@ import ( "github.com/lestrrat-go/strftime" ) -func generateIndex(index string, conf *LogstashFormatSettings, t time.Time) (string, error) { +func generateIndexWithLogstashFormat(index string, conf *LogstashFormatSettings, t time.Time) (string, error) { if conf.Enabled { - partIndex := fmt.Sprintf("%s%s", conf.Prefix, conf.PrefixSeparator) + partIndex := fmt.Sprintf("%s%s", index, conf.PrefixSeparator) var buf bytes.Buffer p, err := strftime.New(fmt.Sprintf("%s%s", partIndex, conf.DateFormat)) if err != nil { diff --git a/exporter/elasticsearchexporter/utils_test.go b/exporter/elasticsearchexporter/utils_test.go index 4311f78ac30c..8d58441edb1c 100644 --- a/exporter/elasticsearchexporter/utils_test.go +++ b/exporter/elasticsearchexporter/utils_test.go @@ -257,23 +257,23 @@ func TestGetSuffixTime(t *testing.T) { defaultCfg := createDefaultConfig().(*Config) defaultCfg.LogstashFormat.Enabled = true testTime := time.Date(2023, 12, 2, 10, 10, 10, 1, time.UTC) - index, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + index, err := generateIndexWithLogstashFormat(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) assert.Nil(t, err) - assert.Equal(t, index, "logstash-2023.12.02") + assert.Equal(t, index, "logs-generic-default-2023.12.02") - defaultCfg.LogstashFormat.Prefix = "otel-logs" + defaultCfg.LogsIndex = "logstash" defaultCfg.LogstashFormat.PrefixSeparator = "." - otelLogsIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + otelLogsIndex, err := generateIndexWithLogstashFormat(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) assert.Nil(t, err) - assert.Equal(t, otelLogsIndex, "otel-logs.2023.12.02") + assert.Equal(t, otelLogsIndex, "logstash.2023.12.02") defaultCfg.LogstashFormat.DateFormat = "%Y-%m-%d" - newOtelLogsIndex, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + newOtelLogsIndex, err := generateIndexWithLogstashFormat(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) assert.Nil(t, err) - assert.Equal(t, newOtelLogsIndex, "otel-logs.2023-12-02") + assert.Equal(t, newOtelLogsIndex, "logstash.2023-12-02") defaultCfg.LogstashFormat.DateFormat = "%d/%m/%Y" - newOtelLogsIndexWithSpecDataFormat, err := generateIndex(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) + newOtelLogsIndexWithSpecDataFormat, err := generateIndexWithLogstashFormat(defaultCfg.LogsIndex, &defaultCfg.LogstashFormat, testTime) assert.Nil(t, err) - assert.Equal(t, newOtelLogsIndexWithSpecDataFormat, "otel-logs.02/12/2023") + assert.Equal(t, newOtelLogsIndexWithSpecDataFormat, "logstash.02/12/2023") } From eebcbbcae96ef56e158d19f8f9ba60cd2b928284 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Tue, 5 Dec 2023 22:51:10 +0800 Subject: [PATCH 08/10] nit error msg. Signed-off-by: Jared Tan --- exporter/elasticsearchexporter/factory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/elasticsearchexporter/factory.go b/exporter/elasticsearchexporter/factory.go index 45d5bc8d2373..8c49dc87ed46 100644 --- a/exporter/elasticsearchexporter/factory.go +++ b/exporter/elasticsearchexporter/factory.go @@ -78,7 +78,7 @@ func createLogsExporter( logsExporter, err := newLogsExporter(set.Logger, cf) if err != nil { - return nil, fmt.Errorf("cannot configure Elasticsearch logs logsExporter: %w", err) + return nil, fmt.Errorf("cannot configure Elasticsearch logsExporter: %w", err) } return exporterhelper.NewLogsExporter( @@ -98,7 +98,7 @@ func createTracesExporter(ctx context.Context, cf := cfg.(*Config) tracesExporter, err := newTracesExporter(set.Logger, cf) if err != nil { - return nil, fmt.Errorf("cannot configure Elasticsearch traces tracesExporter: %w", err) + return nil, fmt.Errorf("cannot configure Elasticsearch tracesExporter: %w", err) } return exporterhelper.NewTracesExporter( ctx, From 87c5b89b94bcdf3f2f7e2cb2ad8e27193ab60212 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Wed, 6 Dec 2023 01:11:43 +0000 Subject: [PATCH 09/10] remove outdated info --- exporter/elasticsearchexporter/logs_exporter.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/exporter/elasticsearchexporter/logs_exporter.go b/exporter/elasticsearchexporter/logs_exporter.go index 3fe9c889d564..589078baeb5f 100644 --- a/exporter/elasticsearchexporter/logs_exporter.go +++ b/exporter/elasticsearchexporter/logs_exporter.go @@ -19,9 +19,7 @@ import ( type elasticsearchLogsExporter struct { logger *zap.Logger - // if logstash format enabled, this index will ignore and won't be used. index string - // if logstash format enabled, exporter will use `logstashIndex` as final index write to. logstashFormat LogstashFormatSettings dynamicIndex bool maxAttempts int From f53b35923644d4d5961c3119159aef966ed706b3 Mon Sep 17 00:00:00 2001 From: Jared Tan Date: Wed, 6 Dec 2023 17:10:52 +0800 Subject: [PATCH 10/10] fix lint Signed-off-by: Jared Tan --- exporter/elasticsearchexporter/logs_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/elasticsearchexporter/logs_exporter.go b/exporter/elasticsearchexporter/logs_exporter.go index 589078baeb5f..b74a5f942ae2 100644 --- a/exporter/elasticsearchexporter/logs_exporter.go +++ b/exporter/elasticsearchexporter/logs_exporter.go @@ -19,7 +19,7 @@ import ( type elasticsearchLogsExporter struct { logger *zap.Logger - index string + index string logstashFormat LogstashFormatSettings dynamicIndex bool maxAttempts int