Skip to content

Commit

Permalink
[connector/spanmetricsv2] Add support for exponential histograms (#94)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lahsivjar
lahsivjar authored Aug 15, 2024
1 parent 4da6290 commit 1442292
Show file tree
Hide file tree
Showing 26 changed files with 1,578 additions and 290 deletions.
50 changes: 37 additions & 13 deletions connector/spanmetricsconnectorv2/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,22 @@ import (
"errors"
"fmt"

"github.com/lightstep/go-expohisto/structure"
"go.opentelemetry.io/collector/confmap"
"go.opentelemetry.io/collector/pdata/pcommon"
)

const (
defaultMetricNameSpans = "trace.span.duration"
defaultMetricDescSpans = "Observed span duration."

// defaultExponentialHistogramMaxSize is the default maximum number
// of buckets per positive or negative number range. 160 buckets
// default supports a high-resolution histogram able to cover a
// long-tail latency distribution from 1ms to 100s with a relative
// error of less than 5%.
// Ref: https://opentelemetry.io/docs/specs/otel/metrics/sdk/#base2-exponential-bucket-histogram-aggregation
defaultExponentialHistogramMaxSize = 160
)

var defaultHistogramBuckets = []float64{
Expand Down Expand Up @@ -90,13 +99,18 @@ type Attribute struct {
}

type Histogram struct {
Explicit *ExplicitHistogram `mapstructure:"explicit"`
Explicit *ExplicitHistogram `mapstructure:"explicit"`
Exponential *ExponentialHistogram `mapstructure:"exponential"`
}

type ExplicitHistogram struct {
Buckets []float64 `mapstructure:"buckets"`
}

type ExponentialHistogram struct {
MaxSize int32 `mapstructure:"max_size"`
}

type Summary struct{}

func (c *Config) Validate() error {
Expand All @@ -111,7 +125,7 @@ func (c *Config) Validate() error {
if info.Unit == "" {
return errors.New("spans: metric unit missing")
}
if info.Histogram.Explicit == nil && info.Summary == nil {
if info.Histogram.Exponential == nil && info.Histogram.Explicit == nil && info.Summary == nil {
return errors.New("metric definition missing, either histogram or summary required")
}
if err := info.validateHistogram(); err != nil {
Expand All @@ -126,11 +140,17 @@ func (c *Config) Validate() error {
}

func (i *MetricInfo) validateHistogram() error {
if i.Histogram.Explicit == nil {
return nil
if i.Histogram.Explicit != nil {
if len(i.Histogram.Explicit.Buckets) == 0 {
return errors.New("histogram buckets missing")
}
}
if len(i.Histogram.Explicit.Buckets) == 0 {
return errors.New("histogram buckets missing")
if i.Histogram.Exponential != nil {
if _, err := structure.NewConfig(
structure.WithMaxSize(i.Histogram.Exponential.MaxSize),
).Validate(); err != nil {
return err
}
}
return nil
}
Expand Down Expand Up @@ -174,17 +194,22 @@ func (c *Config) Unmarshal(componentParser *confmap.Conf) error {
if info.Unit == "" {
info.Unit = MetricUnitMs
}
if info.Histogram.Explicit == nil && info.Summary == nil {
// Default to explicit bound histogram
// TODO: change default to exp histogram once available
info.Histogram.Explicit = &ExplicitHistogram{}
if info.Histogram.Exponential == nil && info.Histogram.Explicit == nil && info.Summary == nil {
info.Histogram.Exponential = &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
}
}
if info.Histogram.Explicit != nil {
// Add default buckets if explicit histogram is defined
if len(info.Histogram.Explicit.Buckets) == 0 {
info.Histogram.Explicit.Buckets = defaultHistogramBuckets[:]
}
}
if info.Histogram.Exponential != nil {
if info.Histogram.Exponential.MaxSize == 0 {
info.Histogram.Exponential.MaxSize = defaultExponentialHistogramMaxSize
}
}
c.Spans[k] = info
}
return nil
Expand All @@ -197,9 +222,8 @@ func defaultSpansConfig() []MetricInfo {
Description: defaultMetricDescSpans,
Unit: MetricUnitMs,
Histogram: Histogram{
// TODO: change default to exp histogram once available
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
Expand Down
26 changes: 25 additions & 1 deletion connector/spanmetricsconnectorv2/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
{
Expand All @@ -64,6 +67,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
{
Expand All @@ -75,13 +81,16 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
},
},
},
{
path: "with_custom_histogram_buckets",
path: "with_custom_histogram_configs",
expected: &Config{
Spans: []MetricInfo{
{
Expand All @@ -92,6 +101,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: []float64{0.001, 0.1, 1, 10},
},
Exponential: &ExponentialHistogram{
MaxSize: 2,
},
},
},
},
Expand All @@ -114,6 +126,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
{
Expand All @@ -125,6 +140,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
},
Expand All @@ -143,6 +161,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
{
Expand All @@ -154,6 +175,9 @@ func TestConfig(t *testing.T) {
Explicit: &ExplicitHistogram{
Buckets: defaultHistogramBuckets[:],
},
Exponential: &ExponentialHistogram{
MaxSize: defaultExponentialHistogramMaxSize,
},
},
},
},
Expand Down
3 changes: 1 addition & 2 deletions connector/spanmetricsconnectorv2/connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ func (sm *spanMetrics) ConsumeTraces(ctx context.Context, td ptrace.Traces) erro
var multiError error
processedMetrics := pmetric.NewMetrics()
processedMetrics.ResourceMetrics().EnsureCapacity(td.ResourceSpans().Len())
// TODO (lahsivjar): add support for exponential histogram
aggregator := aggregator.NewAggregator()
for i := 0; i < td.ResourceSpans().Len(); i++ {
aggregator.Reset()
Expand All @@ -71,7 +70,7 @@ func (sm *spanMetrics) ConsumeTraces(ctx context.Context, td ptrace.Traces) erro
}
}

if aggregator.Size() == 0 {
if aggregator.Empty() {
continue // don't add an empty resource
}

Expand Down
28 changes: 27 additions & 1 deletion connector/spanmetricsconnectorv2/connector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func TestConnector(t *testing.T) {
"with_attributes",
"with_missing_attribute",
"with_missing_attribute_default_value",
"with_custom_histogram_buckets",
"with_custom_histogram_configs",
"with_identical_metric_name_different_attrs",
"with_identical_metric_name_desc_different_attrs",
"with_summary",
Expand Down Expand Up @@ -80,6 +80,7 @@ func TestConnector(t *testing.T) {
require.NoError(t, err)

require.NoError(t, connector.ConsumeTraces(ctx, inputTraces))
require.Len(t, next.AllMetrics(), 1)
assert.NoError(t, pmetrictest.CompareMetrics(
expectedMetrics,
next.AllMetrics()[0],
Expand Down Expand Up @@ -110,6 +111,11 @@ func BenchmarkConnector(b *testing.B) {
Key: "http.response.status_code",
},
},
Histogram: config.Histogram{
Explicit: &config.ExplicitHistogram{},
Exponential: &config.ExponentialHistogram{},
},
Summary: &config.Summary{},
},
{
Name: "db.trace.span.duration",
Expand All @@ -119,6 +125,11 @@ func BenchmarkConnector(b *testing.B) {
Key: "msg.trace.span.duration",
},
},
Histogram: config.Histogram{
Explicit: &config.ExplicitHistogram{},
Exponential: &config.ExponentialHistogram{},
},
Summary: &config.Summary{},
},
{
Name: "msg.trace.span.duration",
Expand All @@ -128,6 +139,11 @@ func BenchmarkConnector(b *testing.B) {
Key: "messaging.system",
},
},
Histogram: config.Histogram{
Explicit: &config.ExplicitHistogram{},
Exponential: &config.ExponentialHistogram{},
},
Summary: &config.Summary{},
},
{
Name: "404.span.duration",
Expand All @@ -137,6 +153,11 @@ func BenchmarkConnector(b *testing.B) {
Key: "404.attribute",
},
},
Histogram: config.Histogram{
Explicit: &config.ExplicitHistogram{},
Exponential: &config.ExponentialHistogram{},
},
Summary: &config.Summary{},
},
{
Name: "404.span.duration.default",
Expand All @@ -147,6 +168,11 @@ func BenchmarkConnector(b *testing.B) {
DefaultValue: "any",
},
},
Histogram: config.Histogram{
Explicit: &config.ExplicitHistogram{},
Exponential: &config.ExponentialHistogram{},
},
Summary: &config.Summary{},
},
},
}
Expand Down
9 changes: 5 additions & 4 deletions connector/spanmetricsconnectorv2/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,11 @@ func createTracesToMetrics(
Name: info.Name,
Description: info.Description,
},
Unit: info.Unit,
Attributes: attrs,
ExplicitHistogram: info.Histogram.Explicit,
Summary: info.Summary,
Unit: info.Unit,
Attributes: attrs,
ExplicitHistogram: info.Histogram.Explicit,
ExponentialHistogram: info.Histogram.Exponential,
Summary: info.Summary,
}
metricDefs = append(metricDefs, md)
}
Expand Down
1 change: 1 addition & 0 deletions connector/spanmetricsconnectorv2/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/elastic/opentelemetry-collector-components/connector/spanmetri
go 1.21.0

require (
github.com/lightstep/go-expohisto v1.0.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.107.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.107.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.107.0
Expand Down
2 changes: 2 additions & 0 deletions connector/spanmetricsconnectorv2/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lightstep/go-expohisto v1.0.0 h1:UPtTS1rGdtehbbAF7o/dhkWLTDI73UifG8LbfQI7cA4=
github.com/lightstep/go-expohisto v1.0.0/go.mod h1:xDXD0++Mu2FOaItXtdDfksfgxfV0z1TMPa+e/EUd0cs=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
Expand Down
Loading

0 comments on commit 1442292

Please sign in to comment.