From af6248b77de9bcb5c8bf1bc1aa4883c95740e556 Mon Sep 17 00:00:00 2001 From: "Gavin Zhang (Kunyuan Zhang)" <31523962+gavindoudou@users.noreply.github.com> Date: Mon, 3 May 2021 13:30:51 -0700 Subject: [PATCH] Add timing/histogram for statsD receiver as OTLP summary (#3261) --- go.sum | 1 + receiver/statsdreceiver/README.md | 20 ++-- receiver/statsdreceiver/config.go | 11 +- receiver/statsdreceiver/config_test.go | 33 +----- receiver/statsdreceiver/factory.go | 6 +- receiver/statsdreceiver/factory_test.go | 2 +- receiver/statsdreceiver/go.mod | 1 + receiver/statsdreceiver/go.sum | 1 + .../protocol/metric_translator.go | 29 +++++ .../protocol/metric_translator_test.go | 34 ++++++ .../statsdreceiver/protocol/statsd_parser.go | 58 +++++++++- .../protocol/statsd_parser_test.go | 109 +++++++++++++++++- receiver/statsdreceiver/testdata/config.yaml | 6 +- 13 files changed, 248 insertions(+), 63 deletions(-) diff --git a/go.sum b/go.sum index f75be7ed65ef..e768c7ee0287 100644 --- a/go.sum +++ b/go.sum @@ -1156,6 +1156,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= diff --git a/receiver/statsdreceiver/README.md b/receiver/statsdreceiver/README.md index 5780589e41d8..41f83f7b3bb7 100644 --- a/receiver/statsdreceiver/README.md +++ b/receiver/statsdreceiver/README.md @@ -23,14 +23,12 @@ The Following settings are optional: - `timer_histogram_mapping:`(default value is below): Specify what OTLP type to convert received timing/histogram data to. -// TODO: can add regex support for `match` later. - -`"match"`, we only support `"*"` now. `"statsd_type"` specifies received Statsd data type. Possible values for this setting are `"timing"`, `"timer"` and `"histogram"`. -`"observer_type"` specifies OTLP data type to convert to. The only supported target data type currently is `"gauge"`, which does not perform any aggregation. -Support for `"summary"` data type is planned to be added in the future. +`"observer_type"` specifies OTLP data type to convert to. We support `"gauge"` and `"summary"`. For `"gauge"`, it does not perform any aggregation. +For `"summary`, the statsD receiver will aggregate to one OTLP summary metric for one metric description(the same metric name with the same tags). It will send percentile 0, 10, 50, 90, 95, 100 to the downstream. +TODO: Add a new option to use a smoothed summary like Promethetheus: https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/3261 Example: @@ -42,11 +40,9 @@ receivers: aggregation_interval: 70s enable_metric_type: true timer_histogram_mapping: - - match: "*" - statsd_type: "histogram" + - statsd_type: "histogram" observer_type: "gauge" - - match: "*" - statsd_type: "timing" + - statsd_type: "timing" observer_type: "gauge" ``` @@ -120,11 +116,9 @@ receivers: aggregation_interval: 60s # default enable_metric_type: false # default timer_histogram_mapping: - - match: "*" - statsd_type: "histogram" + - statsd_type: "histogram" observer_type: "gauge" - - match: "*" - statsd_type: "timing" + - statsd_type: "timing" observer_type: "gauge" exporters: diff --git a/receiver/statsdreceiver/config.go b/receiver/statsdreceiver/config.go index a90199e1f058..663da773c932 100644 --- a/receiver/statsdreceiver/config.go +++ b/receiver/statsdreceiver/config.go @@ -37,9 +37,8 @@ type Config struct { func (c *Config) validate() error { var errors []error - supportMatch := []string{"*"} supportedStatsdType := []string{"timing", "timer", "histogram"} - supportedObserverType := []string{"gauge"} + supportedObserverType := []string{"gauge", "summary"} if c.AggregationInterval <= 0 { errors = append(errors, fmt.Errorf("aggregation_interval must be a positive duration")) @@ -47,14 +46,6 @@ func (c *Config) validate() error { var TimerHistogramMappingMissingObjectName bool for _, eachMap := range c.TimerHistogramMapping { - if eachMap.Match == "" { - TimerHistogramMappingMissingObjectName = true - break - } - - if !protocol.Contains(supportMatch, eachMap.Match) { - errors = append(errors, fmt.Errorf("match is not supported: %s", eachMap.Match)) - } if eachMap.StatsdType == "" { TimerHistogramMappingMissingObjectName = true diff --git a/receiver/statsdreceiver/config_test.go b/receiver/statsdreceiver/config_test.go index aabbf101c896..b88237925c27 100644 --- a/receiver/statsdreceiver/config_test.go +++ b/receiver/statsdreceiver/config_test.go @@ -59,7 +59,7 @@ func TestLoadConfig(t *testing.T) { Transport: "custom_transport", }, AggregationInterval: 70 * time.Second, - TimerHistogramMapping: []protocol.TimerHistogramMapping{{Match: "*", StatsdType: "histogram", ObserverType: "gauge"}, {Match: "*", StatsdType: "timing", ObserverType: "gauge"}}, + TimerHistogramMapping: []protocol.TimerHistogramMapping{{StatsdType: "histogram", ObserverType: "gauge"}, {StatsdType: "timing", ObserverType: "gauge"}}, }, r1) } @@ -73,7 +73,6 @@ func TestValidate(t *testing.T) { const ( negativeAggregationIntervalErr = "aggregation_interval must be a positive duration" noObjectNameErr = "must specify object name for all TimerHistogramMappings" - matchNotSupportErr = "match is not supported: %s" statsdTypeNotSupportErr = "statsd_type is not supported: %s" observerTypeNotSupportErr = "observer_type is not supported: %s" ) @@ -83,28 +82,18 @@ func TestValidate(t *testing.T) { name: "negativeAggregationInterval", cfg: &Config{ AggregationInterval: -1, - TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "*", StatsdType: "timing", ObserverType: "gauge"}, - }, - }, - expectedErr: negativeAggregationIntervalErr, - }, - { - name: "emptyMatch", - cfg: &Config{ - AggregationInterval: 10, TimerHistogramMapping: []protocol.TimerHistogramMapping{ {StatsdType: "timing", ObserverType: "gauge"}, }, }, - expectedErr: noObjectNameErr, + expectedErr: negativeAggregationIntervalErr, }, { name: "emptyStatsdType", cfg: &Config{ AggregationInterval: 10, TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "*", ObserverType: "gauge"}, + {ObserverType: "gauge"}, }, }, expectedErr: noObjectNameErr, @@ -114,27 +103,17 @@ func TestValidate(t *testing.T) { cfg: &Config{ AggregationInterval: 10, TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "*", StatsdType: "timing"}, + {StatsdType: "timing"}, }, }, expectedErr: noObjectNameErr, }, - { - name: "MatchNotSupport", - cfg: &Config{ - AggregationInterval: 10, - TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "aaa", StatsdType: "timing", ObserverType: "gauge"}, - }, - }, - expectedErr: fmt.Sprintf(matchNotSupportErr, "aaa"), - }, { name: "StatsdTypeNotSupport", cfg: &Config{ AggregationInterval: 10, TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "*", StatsdType: "abc", ObserverType: "gauge"}, + {StatsdType: "abc", ObserverType: "gauge"}, }, }, expectedErr: fmt.Sprintf(statsdTypeNotSupportErr, "abc"), @@ -144,7 +123,7 @@ func TestValidate(t *testing.T) { cfg: &Config{ AggregationInterval: 10, TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "*", StatsdType: "timer", ObserverType: "gauge1"}, + {StatsdType: "timer", ObserverType: "gauge1"}, }, }, expectedErr: fmt.Sprintf(observerTypeNotSupportErr, "gauge1"), diff --git a/receiver/statsdreceiver/factory.go b/receiver/statsdreceiver/factory.go index 4218b992ddd0..3548af9a8dd9 100644 --- a/receiver/statsdreceiver/factory.go +++ b/receiver/statsdreceiver/factory.go @@ -36,6 +36,10 @@ const ( defaultEnableMetricType = false ) +var ( + defaultTimerHistogramMapping = []protocol.TimerHistogramMapping{{StatsdType: "timer", ObserverType: "gauge"}, {StatsdType: "histogram", ObserverType: "gauge"}} +) + // NewFactory creates a factory for the StatsD receiver. func NewFactory() component.ReceiverFactory { return receiverhelper.NewFactory( @@ -57,7 +61,7 @@ func createDefaultConfig() config.Receiver { }, AggregationInterval: defaultAggregationInterval, EnableMetricType: defaultEnableMetricType, - TimerHistogramMapping: []protocol.TimerHistogramMapping{{Match: "*", StatsdType: "timer", ObserverType: "gauge"}, {Match: "*", StatsdType: "histogram", ObserverType: "gauge"}}, + TimerHistogramMapping: defaultTimerHistogramMapping, } } diff --git a/receiver/statsdreceiver/factory_test.go b/receiver/statsdreceiver/factory_test.go index fe96c853e4f5..227923062daa 100644 --- a/receiver/statsdreceiver/factory_test.go +++ b/receiver/statsdreceiver/factory_test.go @@ -48,7 +48,7 @@ func TestCreateReceiverWithConfigErr(t *testing.T) { cfg := &Config{ AggregationInterval: -1, TimerHistogramMapping: []protocol.TimerHistogramMapping{ - {Match: "*", StatsdType: "timing", ObserverType: "gauge"}, + {StatsdType: "timing", ObserverType: "gauge"}, }, } receiver, err := createMetricsReceiver( diff --git a/receiver/statsdreceiver/go.mod b/receiver/statsdreceiver/go.mod index 37727047a27c..aeda5723afea 100644 --- a/receiver/statsdreceiver/go.mod +++ b/receiver/statsdreceiver/go.mod @@ -8,6 +8,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.2.0 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/mattn/go-colorable v0.1.7 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe github.com/onsi/ginkgo v1.14.1 // indirect github.com/onsi/gomega v1.10.2 // indirect github.com/pelletier/go-toml v1.8.0 // indirect diff --git a/receiver/statsdreceiver/go.sum b/receiver/statsdreceiver/go.sum index 5892e26eb4b6..8b5f34a64afa 100644 --- a/receiver/statsdreceiver/go.sum +++ b/receiver/statsdreceiver/go.sum @@ -750,6 +750,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= diff --git a/receiver/statsdreceiver/protocol/metric_translator.go b/receiver/statsdreceiver/protocol/metric_translator.go index 90aec7d21881..a06e7c8717c4 100644 --- a/receiver/statsdreceiver/protocol/metric_translator.go +++ b/receiver/statsdreceiver/protocol/metric_translator.go @@ -17,6 +17,7 @@ package protocol import ( "time" + "github.com/montanaflynn/stats" "go.opentelemetry.io/collector/consumer/pdata" ) @@ -58,3 +59,31 @@ func buildGaugeMetric(parsedMetric statsDMetric, timeNow time.Time) pdata.Instru return ilm } + +func buildSummaryMetric(summaryMetric summaryMetric) pdata.InstrumentationLibraryMetrics { + dp := pdata.NewSummaryDataPoint() + dp.SetCount(uint64(len(summaryMetric.summaryPoints))) + sum, _ := stats.Sum(summaryMetric.summaryPoints) + dp.SetSum(sum) + dp.SetTimestamp(pdata.TimestampFromTime(summaryMetric.timeNow)) + + quantile := []float64{0, 10, 50, 90, 95, 100} + for _, v := range quantile { + eachQuantile := pdata.NewValueAtQuantile() + eachQuantile.SetQuantile(v) + eachQuantileValue, _ := stats.PercentileNearestRank(summaryMetric.summaryPoints, v) + eachQuantile.SetValue(eachQuantileValue) + dp.QuantileValues().Append(eachQuantile) + } + + nm := pdata.NewMetric() + nm.SetName(summaryMetric.name) + nm.SetDataType(pdata.MetricDataTypeSummary) + nm.Summary().DataPoints().Append(dp) + + ilm := pdata.NewInstrumentationLibraryMetrics() + ilm.Metrics().Append(nm) + + return ilm + +} diff --git a/receiver/statsdreceiver/protocol/metric_translator_test.go b/receiver/statsdreceiver/protocol/metric_translator_test.go index 93798f5da324..728dde437201 100644 --- a/receiver/statsdreceiver/protocol/metric_translator_test.go +++ b/receiver/statsdreceiver/protocol/metric_translator_test.go @@ -74,3 +74,37 @@ func TestBuildGaugeMetric(t *testing.T) { dp.LabelsMap().Insert("mykey2", "myvalue2") assert.Equal(t, metric, expectedMetrics) } + +func TestBuildSummaryMetric(t *testing.T) { + timeNow := time.Now() + + oneSummaryMetric := summaryMetric{ + name: "testSummary", + summaryPoints: []float64{1, 2, 4, 6, 5, 3}, + labelKeys: []string{"mykey", "mykey2"}, + labelValues: []string{"myvalue", "myvalue2"}, + timeNow: timeNow, + } + + metric := buildSummaryMetric(oneSummaryMetric) + expectedMetric := pdata.NewInstrumentationLibraryMetrics() + expectedMetric.Metrics().Resize(1) + expectedMetric.Metrics().At(0).SetName("testSummary") + expectedMetric.Metrics().At(0).SetDataType(pdata.MetricDataTypeSummary) + expectedMetric.Metrics().At(0).Summary().DataPoints().Resize(1) + expectedMetric.Metrics().At(0).Summary().DataPoints().At(0).SetSum(21) + expectedMetric.Metrics().At(0).Summary().DataPoints().At(0).SetCount(6) + expectedMetric.Metrics().At(0).Summary().DataPoints().At(0).SetTimestamp(pdata.TimestampFromTime(timeNow)) + quantile := []float64{0, 10, 50, 90, 95, 100} + value := []float64{1, 1, 3, 6, 6, 6} + for int, v := range quantile { + eachQuantile := pdata.NewValueAtQuantile() + eachQuantile.SetQuantile(v) + eachQuantileValue := value[int] + eachQuantile.SetValue(eachQuantileValue) + expectedMetric.Metrics().At(0).Summary().DataPoints().At(0).QuantileValues().Append(eachQuantile) + } + + assert.Equal(t, metric, expectedMetric) + +} diff --git a/receiver/statsdreceiver/protocol/statsd_parser.go b/receiver/statsdreceiver/protocol/statsd_parser.go index 7906831d857d..6709baaeeb80 100644 --- a/receiver/statsdreceiver/protocol/statsd_parser.go +++ b/receiver/statsdreceiver/protocol/statsd_parser.go @@ -43,7 +43,6 @@ const ( ) type TimerHistogramMapping struct { - Match string `mapstructure:"match"` StatsdType string `mapstructure:"statsd_type"` ObserverType string `mapstructure:"observer_type"` } @@ -52,12 +51,21 @@ type TimerHistogramMapping struct { type StatsDParser struct { gauges map[statsDMetricdescription]pdata.InstrumentationLibraryMetrics counters map[statsDMetricdescription]pdata.InstrumentationLibraryMetrics + summaries map[statsDMetricdescription]summaryMetric timersAndDistributions []pdata.InstrumentationLibraryMetrics enableMetricType bool observeTimer string observeHistogram string } +type summaryMetric struct { + name string + summaryPoints []float64 + labelKeys []string + labelValues []string + timeNow time.Time +} + type statsDMetric struct { description statsDMetricdescription value string @@ -80,6 +88,8 @@ func (p *StatsDParser) Initialize(enableMetricType bool, sendTimerHistogram []Ti p.gauges = make(map[statsDMetricdescription]pdata.InstrumentationLibraryMetrics) p.counters = make(map[statsDMetricdescription]pdata.InstrumentationLibraryMetrics) p.timersAndDistributions = make([]pdata.InstrumentationLibraryMetrics, 0) + p.summaries = make(map[statsDMetricdescription]summaryMetric) + p.enableMetricType = enableMetricType for _, eachMap := range sendTimerHistogram { switch eachMap.StatsdType { @@ -109,10 +119,14 @@ func (p *StatsDParser) GetMetrics() pdata.Metrics { rm.InstrumentationLibraryMetrics().Append(metric) } + for _, summaryMetric := range p.summaries { + metrics.ResourceMetrics().At(0).InstrumentationLibraryMetrics().Append(buildSummaryMetric(summaryMetric)) + } + p.gauges = make(map[statsDMetricdescription]pdata.InstrumentationLibraryMetrics) p.counters = make(map[statsDMetricdescription]pdata.InstrumentationLibraryMetrics) p.timersAndDistributions = make([]pdata.InstrumentationLibraryMetrics, 0) - + p.summaries = make(map[statsDMetricdescription]summaryMetric) return metrics } @@ -155,12 +169,52 @@ func (p *StatsDParser) Aggregate(line string) error { switch p.observeHistogram { case "gauge": p.timersAndDistributions = append(p.timersAndDistributions, buildGaugeMetric(parsedMetric, timeNowFunc())) + case "summary": + eachSummaryMetric, ok := p.summaries[parsedMetric.description] + if !ok { + p.summaries[parsedMetric.description] = summaryMetric{ + name: parsedMetric.description.name, + summaryPoints: []float64{parsedMetric.floatvalue}, + labelKeys: parsedMetric.labelKeys, + labelValues: parsedMetric.labelValues, + timeNow: timeNowFunc(), + } + } else { + points := eachSummaryMetric.summaryPoints + p.summaries[parsedMetric.description] = summaryMetric{ + name: parsedMetric.description.name, + summaryPoints: append(points, parsedMetric.floatvalue), + labelKeys: parsedMetric.labelKeys, + labelValues: parsedMetric.labelValues, + timeNow: timeNowFunc(), + } + } } case statsdTiming: switch p.observeTimer { case "gauge": p.timersAndDistributions = append(p.timersAndDistributions, buildGaugeMetric(parsedMetric, timeNowFunc())) + case "summary": + eachSummaryMetric, ok := p.summaries[parsedMetric.description] + if !ok { + p.summaries[parsedMetric.description] = summaryMetric{ + name: parsedMetric.description.name, + summaryPoints: []float64{parsedMetric.floatvalue}, + labelKeys: parsedMetric.labelKeys, + labelValues: parsedMetric.labelValues, + timeNow: timeNowFunc(), + } + } else { + points := eachSummaryMetric.summaryPoints + p.summaries[parsedMetric.description] = summaryMetric{ + name: parsedMetric.description.name, + summaryPoints: append(points, parsedMetric.floatvalue), + labelKeys: parsedMetric.labelKeys, + labelValues: parsedMetric.labelValues, + timeNow: timeNowFunc(), + } + } } } diff --git a/receiver/statsdreceiver/protocol/statsd_parser_test.go b/receiver/statsdreceiver/protocol/statsd_parser_test.go index b7ccfc52ce5a..a686857bd185 100644 --- a/receiver/statsdreceiver/protocol/statsd_parser_test.go +++ b/receiver/statsdreceiver/protocol/statsd_parser_test.go @@ -732,7 +732,7 @@ func TestStatsDParser_Aggregate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var err error p := &StatsDParser{} - p.Initialize(false, []TimerHistogramMapping{{Match: "*", StatsdType: "timer", ObserverType: "gauge"}, {Match: "*", StatsdType: "histogram", ObserverType: "gauge"}}) + p.Initialize(false, []TimerHistogramMapping{{StatsdType: "timer", ObserverType: "gauge"}, {StatsdType: "histogram", ObserverType: "gauge"}}) for _, line := range tt.input { err = p.Aggregate(line) } @@ -800,7 +800,7 @@ func TestStatsDParser_AggregateWithMetricType(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var err error p := &StatsDParser{} - p.Initialize(true, []TimerHistogramMapping{{Match: "*", StatsdType: "timer", ObserverType: "gauge"}, {Match: "*", StatsdType: "histogram", ObserverType: "gauge"}}) + p.Initialize(true, []TimerHistogramMapping{{StatsdType: "timer", ObserverType: "gauge"}, {StatsdType: "histogram", ObserverType: "gauge"}}) for _, line := range tt.input { err = p.Aggregate(line) } @@ -814,9 +814,99 @@ func TestStatsDParser_AggregateWithMetricType(t *testing.T) { } } +func TestStatsDParser_AggregateTimmerWithSummary(t *testing.T) { + timeNowFunc = func() time.Time { + return time.Unix(711, 0) + } + + tests := []struct { + name string + input []string + expectedSummaries map[statsDMetricdescription]summaryMetric + err error + }{ + { + name: "timer", + input: []string{ + "statsdTestMetric1:1|ms|#mykey:myvalue", + "statsdTestMetric2:2|ms|#mykey:myvalue", + "statsdTestMetric1:1|ms|#mykey:myvalue", + "statsdTestMetric1:10|ms|#mykey:myvalue", + "statsdTestMetric1:20|ms|#mykey:myvalue", + "statsdTestMetric2:5|ms|#mykey:myvalue", + "statsdTestMetric2:10|ms|#mykey:myvalue", + "statsdTestMetric1:20|ms|@0.1|#mykey:myvalue", + }, + expectedSummaries: map[statsDMetricdescription]summaryMetric{ + testDescription("statsdTestMetric1", "ms", + []string{"mykey"}, []string{"myvalue"}): { + name: "statsdTestMetric1", + summaryPoints: []float64{1, 1, 10, 20, 200}, + labelKeys: []string{"mykey"}, + labelValues: []string{"myvalue"}, + timeNow: timeNowFunc(), + }, + testDescription("statsdTestMetric2", "ms", + []string{"mykey"}, []string{"myvalue"}): { + name: "statsdTestMetric2", + summaryPoints: []float64{2, 5, 10}, + labelKeys: []string{"mykey"}, + labelValues: []string{"myvalue"}, + timeNow: timeNowFunc(), + }, + }, + }, + { + name: "histogram", + input: []string{ + "statsdTestMetric1:1|h|#mykey:myvalue", + "statsdTestMetric2:2|h|#mykey:myvalue", + "statsdTestMetric1:1|h|#mykey:myvalue", + "statsdTestMetric1:10|h|#mykey:myvalue", + "statsdTestMetric1:20|h|#mykey:myvalue", + "statsdTestMetric2:5|h|#mykey:myvalue", + "statsdTestMetric2:10|h|#mykey:myvalue", + }, + expectedSummaries: map[statsDMetricdescription]summaryMetric{ + testDescription("statsdTestMetric1", "h", + []string{"mykey"}, []string{"myvalue"}): { + name: "statsdTestMetric1", + summaryPoints: []float64{1, 1, 10, 20}, + labelKeys: []string{"mykey"}, + labelValues: []string{"myvalue"}, + timeNow: timeNowFunc(), + }, + testDescription("statsdTestMetric2", "h", + []string{"mykey"}, []string{"myvalue"}): { + name: "statsdTestMetric2", + summaryPoints: []float64{2, 5, 10}, + labelKeys: []string{"mykey"}, + labelValues: []string{"myvalue"}, + timeNow: timeNowFunc(), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + p := &StatsDParser{} + p.Initialize(false, []TimerHistogramMapping{{StatsdType: "timer", ObserverType: "summary"}, {StatsdType: "histogram", ObserverType: "summary"}}) + for _, line := range tt.input { + err = p.Aggregate(line) + } + if tt.err != nil { + assert.Equal(t, tt.err, err) + } else { + assert.Equal(t, tt.expectedSummaries, p.summaries) + } + }) + } +} + func TestStatsDParser_Initialize(t *testing.T) { p := &StatsDParser{} - p.Initialize(true, []TimerHistogramMapping{{Match: "*", StatsdType: "timer", ObserverType: "gauge"}, {Match: "*", StatsdType: "histogram", ObserverType: "gauge"}}) + p.Initialize(true, []TimerHistogramMapping{{StatsdType: "timer", ObserverType: "gauge"}, {StatsdType: "histogram", ObserverType: "gauge"}}) labels := attribute.Distinct{} teststatsdDMetricdescription := statsDMetricdescription{ name: "test", @@ -830,7 +920,7 @@ func TestStatsDParser_Initialize(t *testing.T) { func TestStatsDParser_GetMetrics(t *testing.T) { p := &StatsDParser{} - p.Initialize(true, []TimerHistogramMapping{{Match: "*", StatsdType: "timer", ObserverType: "gauge"}, {Match: "*", StatsdType: "histogram", ObserverType: "gauge"}}) + p.Initialize(true, []TimerHistogramMapping{{StatsdType: "timer", ObserverType: "gauge"}, {StatsdType: "histogram", ObserverType: "gauge"}}) p.gauges[testDescription("statsdTestMetric1", "g", []string{"mykey", "metric_type"}, []string{"myvalue", "gauge"})] = buildGaugeMetric(testStatsDMetric("testGauge1", "", 0, 1, false, "g", 0, []string{"mykey", "metric_type"}, []string{"myvalue", "gauge"}), time.Unix(711, 0)) @@ -841,8 +931,17 @@ func TestStatsDParser_GetMetrics(t *testing.T) { []string{"mykey", "metric_type"}, []string{"myvalue", "gauge"})] = buildGaugeMetric(testStatsDMetric("statsdTestMetric1", "", 0, 10102, false, "g", 0, []string{"mykey", "metric_type"}, []string{"myvalue", "gauge"}), time.Unix(711, 0)) p.timersAndDistributions = append(p.timersAndDistributions, buildGaugeMetric(testStatsDMetric("statsdTestMetric1", "", 0, 10102, false, "ms", 0, []string{"mykey2", "metric_type"}, []string{"myvalue2", "gauge"}), time.Unix(711, 0))) + p.summaries = map[statsDMetricdescription]summaryMetric{ + testDescription("statsdTestMetric1", "h", + []string{"mykey"}, []string{"myvalue"}): { + name: "statsdTestMetric1", + summaryPoints: []float64{1, 1, 10, 20}, + labelKeys: []string{"mykey"}, + labelValues: []string{"myvalue"}, + timeNow: timeNowFunc(), + }} metrics := p.GetMetrics() - assert.Equal(t, 4, metrics.ResourceMetrics().At(0).InstrumentationLibraryMetrics().Len()) + assert.Equal(t, 5, metrics.ResourceMetrics().At(0).InstrumentationLibraryMetrics().Len()) } func TestTimeNowFunc(t *testing.T) { diff --git a/receiver/statsdreceiver/testdata/config.yaml b/receiver/statsdreceiver/testdata/config.yaml index c1ef2f21ca5c..684fac02af6e 100644 --- a/receiver/statsdreceiver/testdata/config.yaml +++ b/receiver/statsdreceiver/testdata/config.yaml @@ -6,11 +6,9 @@ receivers: aggregation_interval: 70s enable_metric_type: false timer_histogram_mapping: - - match: "*" - statsd_type: "histogram" + - statsd_type: "histogram" observer_type: "gauge" - - match: "*" - statsd_type: "timing" + - statsd_type: "timing" observer_type: "gauge" processors: