From 7b7dd0f5a0dfa97f18a1d6789ba8ccf0197cd4e8 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Tue, 21 Jul 2020 10:29:07 -0400 Subject: [PATCH] Add support for diffs of MetricData --- internal/goldendataset/metric_gen.go | 16 +- internal/goldendataset/metric_gen_test.go | 1 + testbed/testbed/metric_diff.go | 343 ++++++++++++++++++++++ testbed/testbed/metric_diff_test.go | 105 +++++++ 4 files changed, 460 insertions(+), 5 deletions(-) create mode 100644 testbed/testbed/metric_diff.go create mode 100644 testbed/testbed/metric_diff_test.go diff --git a/internal/goldendataset/metric_gen.go b/internal/goldendataset/metric_gen.go index 9e261d170e0..31392ba82f2 100644 --- a/internal/goldendataset/metric_gen.go +++ b/internal/goldendataset/metric_gen.go @@ -40,6 +40,8 @@ type MetricCfg struct { NumResourceAttrs int // The number of ResourceMetrics for the single MetricData generated NumResourceMetrics int + // The base value for each point + PtVal int // The start time for each point StartTime uint64 // The duration of the steps between each generated point starting at StartTime @@ -55,6 +57,7 @@ func DefaultCfg() MetricCfg { NumPts: 1, NumResourceAttrs: 1, NumResourceMetrics: 1, + PtVal: 1, StartTime: 940000000000000000, StepSize: 42, } @@ -128,7 +131,7 @@ func populateIntPoints(cfg MetricCfg, metric pdata.Metric) { pt := pts.At(i) pt.SetStartTime(pdata.TimestampUnixNano(cfg.StartTime)) pt.SetTimestamp(getTimestamp(cfg.StartTime, cfg.StepSize, i)) - pt.SetValue(1 + int64(i)) + pt.SetValue(int64(cfg.PtVal + i)) populatePtLabels(cfg, pt.LabelsMap()) } } @@ -140,7 +143,7 @@ func populateDblPoints(cfg MetricCfg, metric pdata.Metric) { pt := pts.At(i) pt.SetStartTime(pdata.TimestampUnixNano(cfg.StartTime)) pt.SetTimestamp(getTimestamp(cfg.StartTime, cfg.StepSize, i)) - pt.SetValue(float64(1 + i)) + pt.SetValue(float64(cfg.PtVal + i)) populatePtLabels(cfg, pt.LabelsMap()) } } @@ -156,8 +159,9 @@ func populateHistogramPoints(cfg MetricCfg, metric pdata.Metric) { populatePtLabels(cfg, pt.LabelsMap()) setHistogramBounds(pt, 1, 2, 3, 4, 5) addHistogramVal(pt, 1, ts) - addHistogramVal(pt, 3, ts) - addHistogramVal(pt, 3, ts) + for i := 0; i < cfg.PtVal; i++ { + addHistogramVal(pt, 3, ts) + } addHistogramVal(pt, 5, ts) } } @@ -195,7 +199,9 @@ func populateSummaryPoints(cfg MetricCfg, metric pdata.Metric) { pt.SetTimestamp(getTimestamp(cfg.StartTime, cfg.StepSize, i)) setSummaryPercentiles(pt, 0, 50, 95) addSummaryValue(pt, 55, 0) - addSummaryValue(pt, 70, 1) + for i := 0; i < cfg.PtVal; i++ { + addSummaryValue(pt, 70, 1) + } addSummaryValue(pt, 90, 2) populatePtLabels(cfg, pt.LabelsMap()) } diff --git a/internal/goldendataset/metric_gen_test.go b/internal/goldendataset/metric_gen_test.go index 7a9f106f64e..130c03333c5 100644 --- a/internal/goldendataset/metric_gen_test.go +++ b/internal/goldendataset/metric_gen_test.go @@ -85,6 +85,7 @@ func TestHistogramFunctions(t *testing.T) { func TestGenHistogram(t *testing.T) { cfg := DefaultCfg() cfg.MetricDescriptorType = pdata.MetricTypeHistogram + cfg.PtVal = 2 md := MetricDataFromCfg(cfg) pts := getMetric(md).HistogramDataPoints() pt := pts.At(0) diff --git a/testbed/testbed/metric_diff.go b/testbed/testbed/metric_diff.go new file mode 100644 index 00000000000..4d442388bfa --- /dev/null +++ b/testbed/testbed/metric_diff.go @@ -0,0 +1,343 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testbed + +import ( + "fmt" + "reflect" + + "go.opentelemetry.io/collector/consumer/pdata" + "go.opentelemetry.io/collector/consumer/pdatautil" +) + +// MetricDiff is intended to support producing human-readable diffs between two MetricData structs during +// testing. Two MetricDatas, when compared, could produce a list of MetricDiffs containing all of their +// differences, which could be used to correct the differences between the expected and actual values. +type MetricDiff struct { + expectedValue interface{} + actualValue interface{} + msg string +} + +func (mf MetricDiff) String() string { + return fmt.Sprintf("{msg='%v' expected=[%v] actual=[%v]}\n", mf.msg, mf.expectedValue, mf.actualValue) +} + +func pdmToPDRM(pdm []pdata.Metrics) (out []pdata.ResourceMetrics) { + for _, m := range pdm { + md := pdatautil.MetricsToInternalMetrics(m) + rms := md.ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + rm := rms.At(i) + out = append(out, rm) + } + } + return out +} + +func diffRMSlices(sent []pdata.ResourceMetrics, recd []pdata.ResourceMetrics) []*MetricDiff { + var diffs []*MetricDiff + if len(sent) != len(recd) { + return []*MetricDiff{{ + expectedValue: len(sent), + actualValue: len(recd), + msg: "Sent vs received ResourceMetrics not equal length", + }} + } + for i := 0; i < len(sent); i++ { + sentRM := sent[i] + recdRM := recd[i] + diffs = diffRMs(diffs, sentRM, recdRM) + } + return diffs +} + +func diffRMs(diffs []*MetricDiff, expected pdata.ResourceMetrics, actual pdata.ResourceMetrics) []*MetricDiff { + diffs = diffResource(diffs, expected.Resource(), actual.Resource()) + diffs = diffILMSlice( + diffs, + expected.InstrumentationLibraryMetrics(), + actual.InstrumentationLibraryMetrics(), + ) + return diffs +} + +func diffILMSlice( + diffs []*MetricDiff, + expected pdata.InstrumentationLibraryMetricsSlice, + actual pdata.InstrumentationLibraryMetricsSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, actual.Len(), expected.Len(), "InstrumentationLibraryMetricsSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffILM(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffILM( + diffs []*MetricDiff, + expected pdata.InstrumentationLibraryMetrics, + actual pdata.InstrumentationLibraryMetrics, +) []*MetricDiff { + return diffMetrics(diffs, expected.Metrics(), actual.Metrics()) +} + +func diffMetrics(diffs []*MetricDiff, expected pdata.MetricSlice, actual pdata.MetricSlice) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, actual.Len(), expected.Len(), "MetricSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffMetric(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffMetric(diffs []*MetricDiff, expected pdata.Metric, actual pdata.Metric) []*MetricDiff { + diffs = diffMetricDescriptor(diffs, expected.MetricDescriptor(), actual.MetricDescriptor()) + diffs = diffInt64Pts(diffs, expected.Int64DataPoints(), actual.Int64DataPoints()) + diffs = diffDoublePts(diffs, expected.DoubleDataPoints(), actual.DoubleDataPoints()) + diffs = diffHistogramPts(diffs, expected.HistogramDataPoints(), actual.HistogramDataPoints()) + diffs = diffSummaryPts(diffs, expected.SummaryDataPoints(), actual.SummaryDataPoints()) + return diffs +} + +func diffSummaryPts( + diffs []*MetricDiff, + expected pdata.SummaryDataPointSlice, + actual pdata.SummaryDataPointSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, actual.Len(), expected.Len(), "MetricSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffSummaryPt(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffSummaryPt( + diffs []*MetricDiff, + expected pdata.SummaryDataPoint, + actual pdata.SummaryDataPoint, +) []*MetricDiff { + diffs = diff(diffs, expected.Count(), actual.Count(), "SummaryDataPoint Count") + diffs = diff(diffs, expected.Sum(), actual.Sum(), "SummaryDataPoint Sum") + diffs = diffPercentiles(diffs, expected.ValueAtPercentiles(), actual.ValueAtPercentiles()) + return diffs +} + +func diffPercentiles( + diffs []*MetricDiff, + expected pdata.SummaryValueAtPercentileSlice, + actual pdata.SummaryValueAtPercentileSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, expected.Len(), actual.Len(), "MetricSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffSummaryAtPct(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffSummaryAtPct( + diffs []*MetricDiff, + expected pdata.SummaryValueAtPercentile, + actual pdata.SummaryValueAtPercentile, +) []*MetricDiff { + diffs = diff(diffs, expected.Value(), actual.Value(), "SummaryValueAtPercentile Value") + diffs = diff(diffs, expected.Percentile(), actual.Percentile(), "SummaryValueAtPercentile Percentile") + return diffs +} + +func diffMetricDescriptor( + diffs []*MetricDiff, + expected pdata.MetricDescriptor, + actual pdata.MetricDescriptor, +) []*MetricDiff { + diffs = diff(diffs, expected.Type(), actual.Type(), "MetricDescriptor Type") + diffs = diff(diffs, expected.Name(), actual.Name(), "MetricDescriptor Name") + diffs = diff(diffs, expected.Description(), actual.Description(), "MetricDescriptor Description") + diffs = diff(diffs, expected.Unit(), actual.Unit(), "MetricDescriptor Unit") + return diffs +} + +func diffDoublePts( + diffs []*MetricDiff, + expected pdata.DoubleDataPointSlice, + actual pdata.DoubleDataPointSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, expected.Len(), actual.Len(), "DoubleDataPointSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffDoublePt(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffDoublePt( + diffs []*MetricDiff, + expected pdata.DoubleDataPoint, + actual pdata.DoubleDataPoint, +) []*MetricDiff { + return diff(diffs, expected.Value(), actual.Value(), "DoubleDataPoint value") +} + +func diffHistogramPts( + diffs []*MetricDiff, + expected pdata.HistogramDataPointSlice, + actual pdata.HistogramDataPointSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, expected.Len(), actual.Len(), "HistogramDataPointSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffHistogramPt(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffHistogramPt( + diffs []*MetricDiff, + expected pdata.HistogramDataPoint, + actual pdata.HistogramDataPoint, +) []*MetricDiff { + diffs = diff(diffs, expected.Count(), actual.Count(), "HistogramDataPoint Count") + diffs = diff(diffs, expected.Sum(), actual.Sum(), "HistogramDataPoint Sum") + // todo LabelsMap() + diffs = diffBuckets(diffs, expected.Buckets(), actual.Buckets()) + return diffs +} + +func diffBuckets( + diffs []*MetricDiff, + expected pdata.HistogramBucketSlice, + actual pdata.HistogramBucketSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, expected.Len(), actual.Len(), "HistogramBucketSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffBucket(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffBucket( + diffs []*MetricDiff, + expected pdata.HistogramBucket, + actual pdata.HistogramBucket, +) []*MetricDiff { + diffs = diff(diffs, expected.Count(), actual.Count(), "HistogramBucket Count") + diffs = diffExemplar(diffs, expected.Exemplar(), actual.Exemplar()) + return diffs +} + +func diffExemplar( + diffs []*MetricDiff, + expected pdata.HistogramBucketExemplar, + actual pdata.HistogramBucketExemplar, +) []*MetricDiff { + diffs = diff(diffs, expected.IsNil(), actual.IsNil(), "HistogramBucketExemplar IsNil") + if expected.IsNil() || actual.IsNil() { + return diffs + } + return diff(diffs, expected.Value(), actual.Value(), "HistogramBucketExemplar Value") +} + +func diffInt64Pts( + diffs []*MetricDiff, + expected pdata.Int64DataPointSlice, + actual pdata.Int64DataPointSlice, +) []*MetricDiff { + var mismatch bool + diffs, mismatch = diffValues(diffs, expected.Len(), actual.Len(), "Int64DataPointSlice len") + if mismatch { + return diffs + } + for i := 0; i < expected.Len(); i++ { + diffs = diffInt64Pt(diffs, expected.At(i), actual.At(i)) + } + return diffs +} + +func diffInt64Pt( + diffs []*MetricDiff, + expected pdata.Int64DataPoint, + actual pdata.Int64DataPoint, +) []*MetricDiff { + return diff(diffs, expected.Value(), actual.Value(), "Int64DataPoint value") +} + +func diffResource(diffs []*MetricDiff, expected pdata.Resource, actual pdata.Resource) []*MetricDiff { + return diffAttrs(diffs, expected.Attributes(), actual.Attributes()) +} + +func diffAttrs(diffs []*MetricDiff, expected pdata.AttributeMap, actual pdata.AttributeMap) []*MetricDiff { + if !reflect.DeepEqual(expected, actual) { + diffs = append(diffs, &MetricDiff{ + expectedValue: attrMapToString(expected), + actualValue: attrMapToString(actual), + msg: "Resource attributes", + }) + } + return diffs +} + +func diff(diffs []*MetricDiff, expected interface{}, actual interface{}, msg string) []*MetricDiff { + out, _ := diffValues(diffs, expected, actual, msg) + return out +} + +func diffValues( + diffs []*MetricDiff, + expected interface{}, + actual interface{}, + msg string, +) ([]*MetricDiff, bool) { + if expected != actual { + return append(diffs, &MetricDiff{ + msg: msg, + expectedValue: expected, + actualValue: actual, + }), true + } + return diffs, false +} + +func attrMapToString(m pdata.AttributeMap) string { + out := "" + m.ForEach(func(k string, v pdata.AttributeValue) { + out += "[" + k + "=" + v.StringVal() + "]" + }) + return out +} diff --git a/testbed/testbed/metric_diff_test.go b/testbed/testbed/metric_diff_test.go new file mode 100644 index 00000000000..2f27c944218 --- /dev/null +++ b/testbed/testbed/metric_diff_test.go @@ -0,0 +1,105 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testbed + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/consumer/pdata" + "go.opentelemetry.io/collector/consumer/pdatautil" + "go.opentelemetry.io/collector/internal/data" + "go.opentelemetry.io/collector/internal/goldendataset" +) + +func TestSameMetrics(t *testing.T) { + expected := goldendataset.DefaultMetricData() + actual := goldendataset.DefaultMetricData() + diffs := diffMetricData(expected, actual) + assert.Nil(t, diffs) +} + +func diffMetricData(expected data.MetricData, actual data.MetricData) []*MetricDiff { + expectedRMSlice := expected.ResourceMetrics() + actualRMSlice := actual.ResourceMetrics() + return diffRMSlices(toSlice(expectedRMSlice), toSlice(actualRMSlice)) +} + +func toSlice(s pdata.ResourceMetricsSlice) (out []pdata.ResourceMetrics) { + for i := 0; i < s.Len(); i++ { + out = append(out, s.At(i)) + } + return out +} + +func TestDifferentValues(t *testing.T) { + expected := goldendataset.DefaultMetricData() + cfg := goldendataset.DefaultCfg() + cfg.PtVal = 2 + actual := goldendataset.MetricDataFromCfg(cfg) + diffs := diffMetricData(expected, actual) + assert.Equal(t, 1, len(diffs)) +} + +func TestDifferentNumPts(t *testing.T) { + expected := goldendataset.DefaultMetricData() + cfg := goldendataset.DefaultCfg() + cfg.NumPts = 2 + actual := goldendataset.MetricDataFromCfg(cfg) + diffs := diffMetricData(expected, actual) + assert.Equal(t, 1, len(diffs)) +} + +func TestDifferentPtTypes(t *testing.T) { + expected := goldendataset.DefaultMetricData() + cfg := goldendataset.DefaultCfg() + cfg.MetricDescriptorType = pdata.MetricTypeDouble + actual := goldendataset.MetricDataFromCfg(cfg) + diffs := diffMetricData(expected, actual) + assert.Equal(t, 3, len(diffs)) +} + +func TestHistogram(t *testing.T) { + cfg1 := goldendataset.DefaultCfg() + cfg1.MetricDescriptorType = pdata.MetricTypeHistogram + expected := goldendataset.MetricDataFromCfg(cfg1) + cfg2 := goldendataset.DefaultCfg() + cfg2.MetricDescriptorType = pdata.MetricTypeHistogram + cfg2.PtVal = 2 + actual := goldendataset.MetricDataFromCfg(cfg2) + diffs := diffMetricData(expected, actual) + assert.Equal(t, 3, len(diffs)) +} + +func TestSummary(t *testing.T) { + cfg1 := goldendataset.DefaultCfg() + cfg1.MetricDescriptorType = pdata.MetricTypeSummary + expected := goldendataset.MetricDataFromCfg(cfg1) + cfg2 := goldendataset.DefaultCfg() + cfg2.MetricDescriptorType = pdata.MetricTypeSummary + cfg2.PtVal = 2 + actual := goldendataset.MetricDataFromCfg(cfg2) + diffs := diffMetricData(expected, actual) + assert.Equal(t, 3, len(diffs)) +} + +func TestPDMToPDRM(t *testing.T) { + md := data.NewMetricData() + md.ResourceMetrics().Resize(1) + rm := pdmToPDRM([]pdata.Metrics{pdatautil.MetricsFromInternalMetrics(md)}) + require.Equal(t, 1, len(rm)) +}