From 98de140d2faf9a07c103cedcbbce034801dfbef1 Mon Sep 17 00:00:00 2001 From: Sam Hazlehurst Date: Wed, 13 Mar 2024 17:44:37 -0400 Subject: [PATCH 1/4] Add host metrics generator --- receiver/telemetrygeneratorreceiver/README.md | 97 ++- receiver/telemetrygeneratorreceiver/config.go | 93 ++- .../telemetrygeneratorreceiver/config_test.go | 422 +++++++++++- .../host_metrics_generator.go | 90 ++- .../host_metrics_generator_test.go | 67 +- .../telemetrygeneratorreceiver/receiver.go | 4 +- .../sample_configs/host_metrics_config.yaml | 615 ++++++++++++++++++ .../expected_metrics/host_metrics.yaml | 399 ++++++++++++ 8 files changed, 1754 insertions(+), 33 deletions(-) create mode 100644 receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml create mode 100644 receiver/telemetrygeneratorreceiver/testdata/expected_metrics/host_metrics.yaml diff --git a/receiver/telemetrygeneratorreceiver/README.md b/receiver/telemetrygeneratorreceiver/README.md index 953d7e0d3..33b003cf6 100644 --- a/receiver/telemetrygeneratorreceiver/README.md +++ b/receiver/telemetrygeneratorreceiver/README.md @@ -3,25 +3,22 @@ This receiver is used to generate synthetic telemetry for testing and configurat ## Minimum Agent Versions - Introduced: [v1.46.0](https://github.com/observIQ/bindplane-agent/releases/tag/v1.46.0) +- Updated to include host_metrics: [v1.47.0](https://github.com/observIQ/bindplane-agent/releases/tag/v1.46.0) ## Supported Pipelines - Logs - Metrics - Traces -## How It Works -1. The user configures this receiver in a pipeline. -2. The user configures a supported component to route telemetry from this receiver. - -## Configuration +## Configuration for all generators | Field | Type | Default | Required | Description | |----------------------|-----------|-----------|----------|-------------| | payloads_per_second | int | `1` | `false` | The number of payloads this receiver will generate per second.| | generators | list | | `true` | A list of generators to use.| -### Generator Configuration +### Common Generator Configuration | Field | Type | Default | Required | Description | |----------------------|-----------|------------------|----------|--------------| -| type | string | | `true` | The type of generator to use. Currently only `logs` is supported. | +| type | string | | `true` | The type of generator to use. Currently `logs`, `otlp`, and `host_metrics` are supported. | | resource_attributes | map | | `false` | A map of resource attributes to be included in the generated telemetry. Values can be `any`. | | attributes | map | | `false` | A map of attributes to be included in the generated telemetry. Values can be `any`. | | additional_config | map | | `false` | A map of additional configuration options to be included in the generated telemetry. Values can be `any`.| @@ -32,7 +29,7 @@ This receiver is used to generate synthetic telemetry for testing and configurat | body | string | | `false` | The body of the log, set in additional_config | | severity | int | | `false` | The severity of the log message, set in additional_config | -### Example Configuration +#### Example Configuration ```yaml telemetrygeneratorreceiver: payloads_per_second: 1 @@ -58,3 +55,87 @@ telemetrygeneratorreceiver: body: this is another body severity: 10 ``` + +### OTLP Replay Generator + +The OTLP Replay Generator replays JSON-formatted telemetry from the variable `otlp_json`. It adjusts the timestamps of the telemetry relative the current time, with the most recent record moved to the current time, and the previous records the same relative duration in the past. The `otlp_json` variable should be valid OTLP, such as the JSON created by `plog.JSONMarshaler`,`ptrace.JSONMarshaler`, or `pmetric.JSONMarshaler`. The `otlp_json` variable is set in the `additional_config` section of the generator configuration. + +#### additional_config: + +| Field | Type | Default | Required | Description | +|----------------------|-----------|------------------|----------|--------------| +| telemetry_type | string | | `true` | The type of telemetry to replay: `logs`, `metrics`, or `traces`. | +| otlp_json | string | | `true` | A string of JSON encoded OTLP telemetry| + +#### Example Configuration +```yaml +telemetrygeneratorreceiver: + payloads_per_second: 1 + generators: + - type: otlp + additional_config: + telemetry_type: "metrics", + otlp_json: `{"resourceMetrics":[{"resource":{},"scopeMetrics":[{"scope":{},"metrics":[{"exponentialHistogram":{"dataPoints":[{"attributes":[{"key":"prod-machine","value":{"stringValue":"prod-1"}}],"count":"4","positive":{},"negative":{},"min":0,"max":100}]}}]}]}]}`, +``` + +### Host Metrics Generator + +The host metrics generator creates synthetic host metrics. The generator can be configured to create metrics with arbitrary names, values, and attributes. The generator can be configured to create metrics with a random value between a minimum and maximum value, or a constant value by setting `value_max = value_min`. For `Sum` metrics with unit `s` and `Gauge` metrics, the generator will create a random `float` value. For all other `Sum` metrics, the generator will create a random `int` value. + +#### additional_config: + +| Field | Type | Default | Required | Description | +|----------------------|-----------|------------------|----------|--------------| +| telemetry_type | string | | `true` | The type of telemetry to replay: `logs`, `metrics`, or `traces`. | +| metrics | array | | `true` | A list of host metrics to generate| + +#### metrics: + + +| Field | Type | Default | Required | Description | +|----------------------|-----------|------------------|----------|--------------| +| name | string | | `true` | The metric name | +| value_min | int | | `true` | The metric's minimum value| +| value_max | int | | `true` | The metric's maximum value| +| type | string | | `true` | The metric type: `Gauge`, or `Sum`| +| unit | string | | `true` | The metric unit, either `By`, `by`, `1`, `s`, `{thread}`, `{errors}`, `{packets}`, `{entries}`, `{connections}`, `{faults}`, `{operations}`, or `{processes}`| +| attributes | map | | `false` | A map of attributes to be included in the generated telemetry record. Values can be `any`.| + +#### Example Configuration +```yaml +telemetrygeneratorreceiver: + payloads_per_second: 1 + generators: + - type: host_metrics + resource_attributes: + host.name: 2ed77de7e4c1 + os.type: linux + additional_config: + metrics: + # memory metrics + - name: system.memory.usage + value_min: 100000 + value_max: 1000000000 + type: Sum + unit: By + attributes: + state: cached + # load metrics + - name: system.cpu.load_average.1m + value_min: 0 + value_max: 1 + type: Gauge + unit: "{thread}" + # file system metrics + - name: system.filesystem.usage + value_min: 0 + value_max: 15616700416 + type: Sum + unit: By + attributes: + device: "/dev/vda1" + mode: rw + mountpoint: "/etc/hosts" + state: reserved + type: ext4 +``` \ No newline at end of file diff --git a/receiver/telemetrygeneratorreceiver/config.go b/receiver/telemetrygeneratorreceiver/config.go index 596e5f9ae..0f563f906 100644 --- a/receiver/telemetrygeneratorreceiver/config.go +++ b/receiver/telemetrygeneratorreceiver/config.go @@ -180,7 +180,98 @@ func validateOTLPGenerator(cfg *GeneratorConfig) error { return nil } -func validateHostMetricsGeneratorConfig(_ *GeneratorConfig) error { +func validateHostMetricsGeneratorConfig(g *GeneratorConfig) error { + err := pcommon.NewMap().FromRaw(g.Attributes) + if err != nil { + return fmt.Errorf("error in attributes config: %s", err) + } + + err = pcommon.NewMap().FromRaw(g.ResourceAttributes) + if err != nil { + return fmt.Errorf("error in resource_attributes config: %s", err) + } + + // validate individual metrics + metrics, ok := g.AdditionalConfig["metrics"] + if !ok { + return errors.New("metrics must be set") + } + // check that the metricsArray is a valid array of maps[string]any + // Because of the way the config is unmarshaled, we have to use the `[]any` type + // and then cast to the correct type + metricsArray, ok := metrics.([]any) + if !ok { + return errors.New("metrics must be an array of maps") + } + for _, m := range metricsArray { + metric, ok := m.(map[string]any) + if !ok { + return errors.New("each metric must be a map") + } + // check that the metric has a name + name, ok := metric["name"] + if !ok { + return errors.New("each metric must have a name") + } + // check that the metric has a type + metricType, ok := metric["type"] + if !ok { + return fmt.Errorf("metric %s missing type", name) + } + // check that the metric type is valid + metricTypeStr, ok := metricType.(string) + if !ok { + return fmt.Errorf("metric %s has invalid metric type: %v", name, metricType) + } + switch metricTypeStr { + case "Gauge", "Sum": + default: + return fmt.Errorf("metric %s has invalid metric type: %s", name, metricTypeStr) + } + // check that the metric has a value_min + valMin, ok := metric["value_min"] + if !ok { + return fmt.Errorf("metric %s missing value_min", name) + } + // check that the value_min is a valid int + if _, ok = valMin.(int); !ok { + return fmt.Errorf("metric %s has invalid value_min: %v", name, valMin) + } + // check that the metric has a value_max + valMax, ok := metric["value_max"] + if !ok { + return fmt.Errorf("metric %s missing value_max", name) + } + // check that the value_max is a valid int + if _, ok = valMax.(int); !ok { + return fmt.Errorf("metric %s has invalid value_max: %v", name, valMax) + } + // check that the metric has a unit + unit, ok := metric["unit"] + if !ok { + return fmt.Errorf("metric %s missing unit", name) + } + // check that the unit is a valid string + unitStr, ok := unit.(string) + if !ok { + return fmt.Errorf("metric %s has invalid unit: %v", name, unit) + } + switch unitStr { + case "By", "by", "1", "s", "{thread}", "{errors}", "{packets}", "{entries}", "{connections}", "{faults}", "{operations}", "{processes}": + default: + return fmt.Errorf("metric %s has invalid unit: %s", name, unitStr) + } + + // attributes are optional + if attr, ok := metric["attributes"]; ok { + attributes := attr.(map[string]any) + err = pcommon.NewMap().FromRaw(attributes) + if err != nil { + return fmt.Errorf("error in attributes config for metric %s: %w", name, err) + } + } + } + return nil } diff --git a/receiver/telemetrygeneratorreceiver/config_test.go b/receiver/telemetrygeneratorreceiver/config_test.go index 7ae13fb53..1faec73ca 100644 --- a/receiver/telemetrygeneratorreceiver/config_test.go +++ b/receiver/telemetrygeneratorreceiver/config_test.go @@ -82,21 +82,6 @@ func TestValidate(t *testing.T) { "log_attr2": "log_val2", }, }, - { - Type: "host_metrics", - Attributes: map[string]any{ - "metric_attr1": "metric_val1", - "metric_attr2": "metric_val2", - }, - ResourceAttributes: map[string]any{ - "metric_attr1": "metric_val1", - "metric_attr2": "metric_val2", - }, - AdditionalConfig: map[string]any{ - "metric_attr1": "metric_val1", - "metric_attr2": "metric_val2", - }, - }, { Type: "windows_events", Attributes: map[string]any{ @@ -409,6 +394,413 @@ func TestValidate(t *testing.T) { }, }, }, + { + desc: "host metrics - valid config", + payloads: 1, + errExpected: false, + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "buffered", + }, + }, + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - invalid attributes", + payloads: 1, + errExpected: true, + errText: "error in attributes config: ", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + Attributes: map[string]any{ + "attr_key1": struct{}{}, + }, + }, + }, + }, + { + desc: "host metrics - invalid resource attributes", + payloads: 1, + errExpected: true, + errText: "error in resource_attributes config: ", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + ResourceAttributes: map[string]any{ + "attr_key1": struct{}{}, + }, + }, + }, + }, + { + desc: "host metrics - no metrics", + payloads: 1, + errExpected: true, + errText: "metrics must be set", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + }, + }, + }, + { + desc: "host metrics - metrics not array", + payloads: 1, + errExpected: true, + errText: "metrics must be an array of maps", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - metric not map", + payloads: 1, + errExpected: true, + errText: "each metric must be a map", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + 1, + }, + }, + }, + }, + }, + { + desc: "host metrics - missing name", + payloads: 1, + errExpected: true, + errText: "each metric must have a name", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "buffered", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - missing type", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage missing type", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "unit": "By", + "attributes": map[string]any{ + "state": "buffered", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - invalid type", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage has invalid metric type: 1", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": 1, + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - unknown type", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage has invalid metric type: Foo", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Foo", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - missing value_min", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage missing value_min", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_max": 1000000000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - invalid value_min", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage has invalid value_min: foo", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_max": 1000000000, + "value_min": "foo", + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - missing value_max", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage missing value_max", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - invalid value_max", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage has invalid value_max: foo", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": "foo", + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - missing unit", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage missing unit", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - invalid unit", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage has invalid unit: 1", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": 1, + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - unknown unit", + payloads: 1, + errExpected: true, + errText: "metric system.memory.usage has invalid unit: Foo", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": "Foo", + "attributes": map[string]any{ + "state": "slab_reclaimed", + }, + }, + }, + }, + }, + }, + }, + { + desc: "host metrics - invalid attributes", + payloads: 1, + errExpected: true, + errText: "error in attributes config for metric system.memory.usage: ", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{ + "name": "system.memory.usage", + "value_min": 100000, + "value_max": 1000000000, + "type": "Sum", + "unit": "By", + "attributes": map[string]any{ + "state": struct{}{}, + }, + }, + }, + }, + }, + }, + }, } for _, tc := range testCases { diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go index aaf8b2dc7..6b9c24ade 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go @@ -15,24 +15,102 @@ package telemetrygeneratorreceiver //import "github.com/observiq/bindplane-agent/receiver/telemetrygeneratorreceiver" import ( + "math" + "math/rand" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" ) // hostMetricsGenerator is a generator for host metrics. It generates a sampling of host metrics // emulating the Host Metrics receiver: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver type hostMetricsGenerator struct { - cfg GeneratorConfig - logger *zap.Logger + otlpGenerator + // dataPointUpdaters is a list of functions that update the data points with new values + // for each batch of metrics that are generated + dataPointUpdaters []func() } func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerator { - return &hostMetricsGenerator{ - cfg: cfg, - logger: logger, + + g := &hostMetricsGenerator{ + otlpGenerator: otlpGenerator{ + cfg: cfg, + logger: logger, + logs: plog.NewLogs(), + metrics: pmetric.NewMetrics(), + traces: ptrace.NewTraces(), + }, } + // Load up Resources attributes + + newResource := g.metrics.ResourceMetrics().AppendEmpty() + newResource.Resource().Attributes().FromRaw(cfg.ResourceAttributes) + + metrics := cfg.AdditionalConfig["metrics"].([]any) + newScope := newResource.ScopeMetrics().AppendEmpty() + for _, m := range metrics { + metric := m.(map[string]any) + var attributes map[string]any + // attributes are optional + if attr, ok := metric["attributes"]; ok { + attributes = attr.(map[string]any) + } + + metricType := metric["type"].(string) + + name := metric["name"].(string) + valueMin := metric["value_min"].(int) + valueMax := metric["value_max"].(int) + unit := metric["unit"].(string) + + newMetric := newScope.Metrics().AppendEmpty() + newMetric.SetUnit(unit) + newMetric.SetName(name) + + // all the host metrics are either sums or gauges + switch metricType { + case "Gauge": + newMetric.SetEmptyGauge() + dp := newMetric.Gauge().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + dp.Attributes().FromRaw(attributes) + + // All the host metric Gauges are float64 + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(getRandomFloat64(valueMin, valueMax)) }) + case "Sum": + newMetric.SetEmptySum() + dp := newMetric.Sum().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + dp.Attributes().FromRaw(attributes) + switch unit { + case "s": + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(float64(math.Trunc(getRandomFloat64(valueMin, valueMax)*100)) / 100) }) + default: + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetIntValue(int64(getRandomFloat64(valueMin, valueMax))) }) + } + } + } + + g.adjustMetricTimes() + + return g +} + +// this is a variable so that it can be overridden in tests +var getRandomFloat64 = func(value_min, value_max int) float64 { + return rand.Float64()*(float64(value_max)-float64(value_min)) + float64(value_min) } func (g *hostMetricsGenerator) generateMetrics() pmetric.Metrics { - return pmetric.NewMetrics() + // Update the data points with new values + for _, updater := range g.dataPointUpdaters { + updater() + } + return g.otlpGenerator.generateMetrics() } diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go index b2cb2276f..da3bc5060 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go @@ -15,12 +15,18 @@ package telemetrygeneratorreceiver import ( + "path/filepath" "testing" + "time" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" "github.com/stretchr/testify/require" "go.uber.org/zap" ) +var expectedHostMetricsDir = filepath.Join("testdata", "expected_metrics") + func TestHostMetricsGenerator(t *testing.T) { test := []struct { @@ -28,14 +34,71 @@ func TestHostMetricsGenerator(t *testing.T) { cfg GeneratorConfig expectedFile string }{ - // TODO tests + { + name: "default", + cfg: GeneratorConfig{ + + Type: "host_metrics", + ResourceAttributes: map[string]any{ + "host.name": "2ed77de7e4c1", + "os.type": "linux", + }, + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_unreclaimable"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "cached"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_reclaimable"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "buffered"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "buffered"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_reclaimable"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_unreclaimable"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "cached"}}, + map[string]any{"name": "system.cpu.load_average.1m", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "{thread}"}, + map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "reserved", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, + map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, + map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, + map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, + map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.conntrack.max", "value_min": 65536, "value_max": 65536, "type": "Sum", "unit": "{entries}"}, + map[string]any{"name": "system.network.conntrack.count", "value_min": 8, "value_max": 64, "type": "Sum", "unit": "{entries}"}, + map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "ESTABLISHED"}}, + map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "LISTEN"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "user", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "system", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "idle", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "interrupt", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "nice", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "softirq", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "steal", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "wait", "cpu": "cpu0"}}, + }, + }, + }, + expectedFile: filepath.Join(expectedHostMetricsDir, "host_metrics.yaml"), + }, } for _, tc := range test { + getRandomFloat64 = func(_, _ int) float64 { + return 0 + } + getCurrentTime = func() time.Time { + return time.Unix(0, 0) + } t.Run(tc.name, func(t *testing.T) { g := newHostMetricsGenerator(tc.cfg, zap.NewNop()) metrics := g.generateMetrics() - require.Equal(t, 0, metrics.MetricCount()) + expectedMetrics, err := golden.ReadMetrics(tc.expectedFile) + require.NoError(t, err) + err = pmetrictest.CompareMetrics(expectedMetrics, metrics) + require.NoError(t, err) }) } } diff --git a/receiver/telemetrygeneratorreceiver/receiver.go b/receiver/telemetrygeneratorreceiver/receiver.go index a2c6b9e45..864475572 100644 --- a/receiver/telemetrygeneratorreceiver/receiver.go +++ b/receiver/telemetrygeneratorreceiver/receiver.go @@ -54,7 +54,9 @@ func newTelemetryGeneratorReceiver(ctx context.Context, logger *zap.Logger, cfg // Shutdown shuts down the telemetry generator receiver func (r *telemetryGeneratorReceiver) Shutdown(ctx context.Context) error { - r.cancelFunc(errors.New("shutdown")) + if r.cancelFunc != nil { + r.cancelFunc(errors.New("shutdown")) + } var err error select { case <-ctx.Done(): diff --git a/receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml b/receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml new file mode 100644 index 000000000..546b7b627 --- /dev/null +++ b/receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml @@ -0,0 +1,615 @@ +telemetrygeneratorreceiver: + payloads_per_second: 1 + generators: + - type: host_metrics + resource_attributes: + host.name: 2ed77de7e4c1 + os.type: linux + additional_config: + metrics: +# memory metrics + - name: system.memory.utilization + value_min: 0 + value_max: 100 + type: Gauge + unit: "1" + attributes: + state: slab_unreclaimable + - name: system.memory.utilization + value_min: 0 + value_max: 100 + type: Gauge + unit: "1" + attributes: + state: cached + - name: system.memory.utilization + value_min: 0 + value_max: 100 + type: Gauge + unit: "1" + attributes: + state: slab_reclaimable + - name: system.memory.utilization + value_min: 0 + value_max: 100 + type: Gauge + unit: "1" + attributes: + state: buffered + - name: system.memory.usage + value_min: 100000 + value_max: 1000000000 + type: Sum + unit: By + attributes: + state: buffered + - name: system.memory.usage + value_min: 100000 + value_max: 1000000000 + type: Sum + unit: By + attributes: + state: slab_reclaimable + - name: system.memory.usage + value_min: 100000 + value_max: 1000000000 + type: Sum + unit: By + attributes: + state: slab_unreclaimable + - name: system.memory.usage + value_min: 100000 + value_max: 1000000000 + type: Sum + unit: By + attributes: + state: cached +# load metrics + - name: system.cpu.load_average.1m + value_min: 0 + value_max: 1 + type: Gauge + unit: "{thread}" +# file system metrics + - name: system.filesystem.usage + value_min: 0 + value_max: 15616700416 + type: Sum + unit: By + attributes: + device: "/dev/vda1" + mode: rw + mountpoint: "/etc/hosts" + state: reserved + type: ext4 + - name: system.filesystem.usage + value_min: 0 + value_max: 15616700416 + type: Sum + unit: By + attributes: + device: "/dev/vda1" + mode: rw + mountpoint: "/etc/hosts" + state: free + type: ext4 + - name: system.filesystem.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + device: "/dev/vda1" + mode: rw + mountpoint: "/etc/hosts" + state: free + type: ext4 + - name: system.filesystem.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + device: "/dev/vda1" + mode: rw + mountpoint: "/etc/hosts" + state: free + type: ext4 +# network metrics + - name: system.network.packets + value_min: 0 + value_max: 1000000 + type: Sum + unit: "{packets}" + attributes: + device: eth0 + direction: receive + - name: system.network.packets + value_min: 0 + value_max: 1000000 + type: Sum + unit: "{packets}" + attributes: + device: eth0 + direction: send + - name: system.network.io + value_min: 0 + value_max: 100000000 + type: Sum + unit: By + attributes: + device: eth0 + direction: send + - name: system.network.io + value_min: 0 + value_max: 100000000 + type: Sum + unit: By + attributes: + device: eth0 + direction: receive + - name: system.network.errors + value_min: 0 + value_max: 1000 + type: Sum + unit: "{errors}" + attributes: + device: eth0 + direction: receive + - name: system.network.errors + value_min: 0 + value_max: 1000 + type: Sum + unit: "{errors}" + attributes: + device: eth0 + direction: transmit + - name: system.network.dropped + value_min: 0 + value_max: 1000 + type: Sum + unit: "{packets}" + attributes: + device: eth0 + direction: transmit + - name: system.network.dropped + value_min: 0 + value_max: 1000 + type: Sum + unit: "{packets}" + attributes: + device: eth0 + direction: receive + - name: system.network.conntrack.max + value_min: 65536 + value_max: 65536 + type: Sum + unit: "{entries}" + - name: system.network.conntrack.count + value_min: 8 + value_max: 64 + type: Sum + unit: "{entries}" + - name: system.network.connections + value_min: 0 + value_max: 64 + type: Sum + unit: "{connections}" + attributes: + protocol: tcp + state: ESTABLISHED + - name: system.network.connections + value_min: 0 + value_max: 64 + type: Sum + unit: "{connections}" + attributes: + protocol: tcp + state: LISTEN +# paging metrics + - name: system.paging.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + device: "/swap" + state: free + - name: system.paging.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + device: "/swap" + state: used + - name: system.paging.usage + value_min: 0 + value_max: 1000000000000 + type: Sum + unit: By + attributes: + device: "/swap" + state: free + - name: system.paging.usage + value_min: 0 + value_max: 1000000000000 + type: Sum + unit: By + attributes: + device: "/swap" + state: used + - name: system.paging.operations + value_min: 0 + value_max: 1000000000000 + type: Sum + unit: "{operations}" + attributes: + direction: page_in + type: minor + - name: system.paging.operations + value_min: 0 + value_max: 1000000000000 + type: Sum + unit: "{operations}" + attributes: + direction: page_out + type: minor + - name: system.paging.operations + value_min: 0 + value_max: 10000000 + type: Sum + unit: "{operations}" + attributes: + direction: page_in + type: major + - name: system.paging.operations + value_min: 0 + value_max: 10000000 + type: Sum + unit: "{operations}" + attributes: + direction: page_out + type: major + - name: system.paging.faults + value_min: 0 + value_max: 1000000000000 + type: Sum + unit: "{faults}" + attributes: + type: major + - name: system.paging.faults + value_min: 0 + value_max: 1000000000000 + type: Sum + unit: "{faults}" + attributes: + type: major +# process metrics + - name: system.processes.created + value_min: 0 + value_max: 100000 + type: Sum + unit: "{processes}" + - name: system.processes.count + value_min: 0 + value_max: 1000 + type: Sum + unit: "{processes}" + attributes: + status: unknown + - name: system.processes.count + value_min: 0 + value_max: 100 + type: Sum + unit: "{processes}" + attributes: + status: sleeping + - name: system.processes.count + value_min: 0 + value_max: 10 + type: Sum + unit: "{processes}" + attributes: + status: running +# cpu metrics + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: user + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: system + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: idle + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: interrupt + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: nice + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: softirq + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: steal + cpu: cpu3 + - name: system.cpu.utilization + value_min: 0 + value_max: 1 + type: Gauge + unit: "1" + attributes: + state: wait + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: user + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: system + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: idle + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: interrupt + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: nice + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: softirq + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: steal + cpu: cpu3 + - name: system.cpu.time + value_min: 0 + value_max: 10000 + type: Sum + unit: s + attributes: + state: wait + cpu: cpu3 +# disk metrics + - name: system.disk.weighted_io_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: loop0 + - name: system.disk.weighted_io_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: vda + - name: system.disk.io_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: loop0 + - name: system.disk.io_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: vda + - name: system.disk.io + value_min: 0 + value_max: 1000000000 + type: Sum + unit: By + attributes: + device: loop0 + direction: read + - name: system.disk.io + value_min: 0 + value_max: 1000000000 + type: Sum + unit: By + attributes: + direction: read + device: vda + - name: system.disk.io + value_min: 0 + value_max: 1000000000 + type: Sum + unit: By + attributes: + device: loop0 + direction: write + - name: system.disk.io + value_min: 0 + value_max: 1000000000 + type: Sum + unit: By + attributes: + direction: write + device: vda + - name: system.disk.pending_operations + value_min: 0 + value_max: 10 + type: Sum + unit: "{operations}" + attributes: + device: loop0 + - name: system.disk.pending_operations + value_min: 0 + value_max: 10 + type: Sum + unit: "{operations}" + attributes: + device: vda + - name: system.disk.operations + value_min: 0 + value_max: 1000 + type: Sum + unit: "{operations}" + attributes: + device: loop0 + direction: read + - name: system.disk.operations + value_min: 0 + value_max: 1000 + type: Sum + unit: "{operations}" + attributes: + device: vda + direction: read + - name: system.disk.operations + value_min: 0 + value_max: 1000 + type: Sum + unit: "{operations}" + attributes: + device: loop0 + direction: write + - name: system.disk.operations + value_min: 0 + value_max: 1000 + type: Sum + unit: "{operations}" + attributes: + device: vda + direction: write + - name: system.disk.merged + value_min: 0 + value_max: 100000 + type: Sum + unit: "{operations}" + attributes: + device: loop0 + direction: read + - name: system.disk.merged + value_min: 0 + value_max: 100000 + type: Sum + unit: "{operations}" + attributes: + device: vda + direction: read + - name: system.disk.merged + value_min: 0 + value_max: 100000 + type: Sum + unit: "{operations}" + attributes: + device: loop0 + direction: write + - name: system.disk.merged + value_min: 0 + value_max: 100000 + type: Sum + unit: "{operations}" + attributes: + device: vda + direction: write + - name: system.disk.operation_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: loop0 + direction: read + - name: system.disk.operation_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: vda + direction: read + - name: system.disk.operation_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: loop0 + direction: write + - name: system.disk.operation_time + value_min: 0 + value_max: 1000 + type: Sum + unit: s + attributes: + device: vda + direction: write + diff --git a/receiver/telemetrygeneratorreceiver/testdata/expected_metrics/host_metrics.yaml b/receiver/telemetrygeneratorreceiver/testdata/expected_metrics/host_metrics.yaml new file mode 100644 index 000000000..1ec32457a --- /dev/null +++ b/receiver/telemetrygeneratorreceiver/testdata/expected_metrics/host_metrics.yaml @@ -0,0 +1,399 @@ +resourceMetrics: + - resource: + attributes: + - key: host.name + value: + stringValue: 2ed77de7e4c1 + - key: os.type + value: + stringValue: linux + scopeMetrics: + - metrics: + - gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: state + value: + stringValue: slab_unreclaimable + name: system.memory.utilization + unit: "1" + - gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: state + value: + stringValue: cached + name: system.memory.utilization + unit: "1" + - gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: state + value: + stringValue: slab_reclaimable + name: system.memory.utilization + unit: "1" + - gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: state + value: + stringValue: buffered + name: system.memory.utilization + unit: "1" + - name: system.memory.usage + sum: + dataPoints: + - asInt: "0" + attributes: + - key: state + value: + stringValue: buffered + unit: By + - name: system.memory.usage + sum: + dataPoints: + - asInt: "0" + attributes: + - key: state + value: + stringValue: slab_reclaimable + unit: By + - name: system.memory.usage + sum: + dataPoints: + - asInt: "0" + attributes: + - key: state + value: + stringValue: slab_unreclaimable + unit: By + - name: system.memory.usage + sum: + dataPoints: + - asInt: "0" + attributes: + - key: state + value: + stringValue: cached + unit: By + - gauge: + dataPoints: + - asDouble: 0 + name: system.cpu.load_average.1m + unit: '{thread}' + - name: system.filesystem.usage + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: /dev/vda1 + - key: mode + value: + stringValue: rw + - key: mountpoint + value: + stringValue: /etc/hosts + - key: state + value: + stringValue: reserved + - key: type + value: + stringValue: ext4 + unit: By + - name: system.filesystem.usage + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: /dev/vda1 + - key: mode + value: + stringValue: rw + - key: mountpoint + value: + stringValue: /etc/hosts + - key: state + value: + stringValue: free + - key: type + value: + stringValue: ext4 + unit: By + - gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: device + value: + stringValue: /dev/vda1 + - key: mode + value: + stringValue: rw + - key: mountpoint + value: + stringValue: /etc/hosts + - key: state + value: + stringValue: free + - key: type + value: + stringValue: ext4 + name: system.filesystem.utilization + unit: "1" + - gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: device + value: + stringValue: /dev/vda1 + - key: mode + value: + stringValue: rw + - key: mountpoint + value: + stringValue: /etc/hosts + - key: state + value: + stringValue: free + - key: type + value: + stringValue: ext4 + name: system.filesystem.utilization + unit: "1" + - name: system.network.packets + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: receive + unit: '{packets}' + - name: system.network.packets + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: send + unit: '{packets}' + - name: system.network.io + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: send + unit: By + - name: system.network.io + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: receive + unit: By + - name: system.network.errors + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: receive + unit: '{errors}' + - name: system.network.errors + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: transmit + unit: '{errors}' + - name: system.network.dropped + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: transmit + unit: '{packets}' + - name: system.network.dropped + sum: + dataPoints: + - asInt: "0" + attributes: + - key: device + value: + stringValue: eth0 + - key: direction + value: + stringValue: receive + unit: '{packets}' + - name: system.network.conntrack.max + sum: + dataPoints: + - asInt: "0" + unit: '{entries}' + - name: system.network.conntrack.count + sum: + dataPoints: + - asInt: "0" + unit: '{entries}' + - name: system.network.connections + sum: + dataPoints: + - asInt: "0" + attributes: + - key: protocol + value: + stringValue: tcp + - key: state + value: + stringValue: ESTABLISHED + unit: '{connections}' + - name: system.network.connections + sum: + dataPoints: + - asInt: "0" + attributes: + - key: protocol + value: + stringValue: tcp + - key: state + value: + stringValue: LISTEN + unit: '{connections}' + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: user + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: system + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: idle + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: interrupt + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: nice + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: softirq + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: steal + unit: s + - name: system.cpu.time + sum: + dataPoints: + - asDouble: 0 + attributes: + - key: cpu + value: + stringValue: cpu0 + - key: state + value: + stringValue: wait + unit: s + scope: {} From 766504e32cbdf2d64bf5e3622a7562ccf45e948d Mon Sep 17 00:00:00 2001 From: Sam Hazlehurst Date: Thu, 14 Mar 2024 08:58:58 -0400 Subject: [PATCH 2/4] Handle errors, gosec exception --- .../host_metrics_generator.go | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go index 6b9c24ade..31b3ee25c 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go @@ -48,7 +48,11 @@ func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGene // Load up Resources attributes newResource := g.metrics.ResourceMetrics().AppendEmpty() - newResource.Resource().Attributes().FromRaw(cfg.ResourceAttributes) + err := newResource.Resource().Attributes().FromRaw(cfg.ResourceAttributes) + if err != nil { + // should be caught in validation + logger.Error("Error setting resource attributes in host_metrics", zap.Error(err)) + } metrics := cfg.AdditionalConfig["metrics"].([]any) newScope := newResource.ScopeMetrics().AppendEmpty() @@ -78,7 +82,12 @@ func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGene dp := newMetric.Gauge().DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - dp.Attributes().FromRaw(attributes) + + err = dp.Attributes().FromRaw(attributes) + if err != nil { + // should be caught in validation + logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) + } // All the host metric Gauges are float64 g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(getRandomFloat64(valueMin, valueMax)) }) @@ -87,7 +96,11 @@ func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGene dp := newMetric.Sum().DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - dp.Attributes().FromRaw(attributes) + err = dp.Attributes().FromRaw(attributes) + if err != nil { + // should be caught in validation + logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) + } switch unit { case "s": g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(float64(math.Trunc(getRandomFloat64(valueMin, valueMax)*100)) / 100) }) @@ -104,6 +117,7 @@ func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGene // this is a variable so that it can be overridden in tests var getRandomFloat64 = func(value_min, value_max int) float64 { + // #nosec G404 - we don't need a cryptographically strong random number generator here return rand.Float64()*(float64(value_max)-float64(value_min)) + float64(value_min) } From a1c6d34600e1f5601f82261eef809c0f65582f38 Mon Sep 17 00:00:00 2001 From: Sam Hazlehurst Date: Fri, 15 Mar 2024 10:33:15 -0400 Subject: [PATCH 3/4] Make host_metrics generator a default configuration of the more flexible metrics generator --- receiver/telemetrygeneratorreceiver/README.md | 28 +++- receiver/telemetrygeneratorreceiver/config.go | 12 +- .../telemetrygeneratorreceiver/config_test.go | 100 +++++++----- .../telemetrygeneratorreceiver/generators.go | 5 + .../host_metrics_generator.go | 146 ++++++------------ .../host_metrics_generator_test.go | 87 ++--------- .../metrics_generator.go | 130 ++++++++++++++++ .../metrics_generator_test.go | 104 +++++++++++++ ...etrics_config.yaml => metrics_config.yaml} | 2 +- .../{host_metrics.yaml => metrics.yaml} | 0 10 files changed, 393 insertions(+), 221 deletions(-) create mode 100644 receiver/telemetrygeneratorreceiver/metrics_generator.go create mode 100644 receiver/telemetrygeneratorreceiver/metrics_generator_test.go rename receiver/telemetrygeneratorreceiver/sample_configs/{host_metrics_config.yaml => metrics_config.yaml} (99%) rename receiver/telemetrygeneratorreceiver/testdata/expected_metrics/{host_metrics.yaml => metrics.yaml} (100%) diff --git a/receiver/telemetrygeneratorreceiver/README.md b/receiver/telemetrygeneratorreceiver/README.md index 33b003cf6..16af37704 100644 --- a/receiver/telemetrygeneratorreceiver/README.md +++ b/receiver/telemetrygeneratorreceiver/README.md @@ -18,7 +18,7 @@ This receiver is used to generate synthetic telemetry for testing and configurat ### Common Generator Configuration | Field | Type | Default | Required | Description | |----------------------|-----------|------------------|----------|--------------| -| type | string | | `true` | The type of generator to use. Currently `logs`, `otlp`, and `host_metrics` are supported. | +| type | string | | `true` | The type of generator to use. Currently `logs`, `otlp`, `metrics`, and `host_metrics` are supported. | | resource_attributes | map | | `false` | A map of resource attributes to be included in the generated telemetry. Values can be `any`. | | attributes | map | | `false` | A map of attributes to be included in the generated telemetry. Values can be `any`. | | additional_config | map | | `false` | A map of additional configuration options to be included in the generated telemetry. Values can be `any`.| @@ -78,16 +78,16 @@ telemetrygeneratorreceiver: otlp_json: `{"resourceMetrics":[{"resource":{},"scopeMetrics":[{"scope":{},"metrics":[{"exponentialHistogram":{"dataPoints":[{"attributes":[{"key":"prod-machine","value":{"stringValue":"prod-1"}}],"count":"4","positive":{},"negative":{},"min":0,"max":100}]}}]}]}]}`, ``` -### Host Metrics Generator +### Metrics Generator -The host metrics generator creates synthetic host metrics. The generator can be configured to create metrics with arbitrary names, values, and attributes. The generator can be configured to create metrics with a random value between a minimum and maximum value, or a constant value by setting `value_max = value_min`. For `Sum` metrics with unit `s` and `Gauge` metrics, the generator will create a random `float` value. For all other `Sum` metrics, the generator will create a random `int` value. +The metrics generator creates synthetic metrics. The generator can be configured to create metrics with arbitrary names, values, and attributes. The generator can be configured to create metrics with a random value between a minimum and maximum value, or a constant value by setting `value_max = value_min`. For `Sum` metrics with unit `s` and `Gauge` metrics, the generator will create a random `float` value. For all other `Sum` metrics, the generator will create a random `int` value. #### additional_config: | Field | Type | Default | Required | Description | |----------------------|-----------|------------------|----------|--------------| | telemetry_type | string | | `true` | The type of telemetry to replay: `logs`, `metrics`, or `traces`. | -| metrics | array | | `true` | A list of host metrics to generate| +| metrics | array | | `true` | A list of metrics to generate| #### metrics: @@ -106,7 +106,7 @@ The host metrics generator creates synthetic host metrics. The generator can be telemetrygeneratorreceiver: payloads_per_second: 1 generators: - - type: host_metrics + - type: metrics resource_attributes: host.name: 2ed77de7e4c1 os.type: linux @@ -138,4 +138,20 @@ telemetrygeneratorreceiver: mountpoint: "/etc/hosts" state: reserved type: ext4 -``` \ No newline at end of file +``` + + +### Host Metrics Generator + +The host metrics generator creates synthetic host metrics, from a list of pre-defined metrics. The metrics resource attributes can be set in the `resource_attributes` section of the generator configuration. + +#### Example Configuration +```yaml +telemetrygeneratorreceiver: + payloads_per_second: 1 + generators: + - type: host_metrics + resource_attributes: + host.name: 2ed77de7e4c1 + os.type: linux +``` \ No newline at end of file diff --git a/receiver/telemetrygeneratorreceiver/config.go b/receiver/telemetrygeneratorreceiver/config.go index 0f563f906..33231938d 100644 --- a/receiver/telemetrygeneratorreceiver/config.go +++ b/receiver/telemetrygeneratorreceiver/config.go @@ -69,6 +69,8 @@ func (g *GeneratorConfig) Validate() error { switch g.Type { case generatorTypeLogs: return validateLogGeneratorConfig(g) + case generatorTypeMetrics: + return validateMetricsGeneratorConfig(g) case generatorTypeHostMetrics: return validateHostMetricsGeneratorConfig(g) case generatorTypeWindowsEvents: @@ -180,7 +182,7 @@ func validateOTLPGenerator(cfg *GeneratorConfig) error { return nil } -func validateHostMetricsGeneratorConfig(g *GeneratorConfig) error { +func validateMetricsGeneratorConfig(g *GeneratorConfig) error { err := pcommon.NewMap().FromRaw(g.Attributes) if err != nil { return fmt.Errorf("error in attributes config: %s", err) @@ -275,6 +277,14 @@ func validateHostMetricsGeneratorConfig(g *GeneratorConfig) error { return nil } +func validateHostMetricsGeneratorConfig(g *GeneratorConfig) error { + err := pcommon.NewMap().FromRaw(g.ResourceAttributes) + if err != nil { + return fmt.Errorf("error in resource_attributes config: %s", err) + } + return nil +} + func validateWindowsEventsGeneratorConfig(_ *GeneratorConfig) error { return nil } diff --git a/receiver/telemetrygeneratorreceiver/config_test.go b/receiver/telemetrygeneratorreceiver/config_test.go index 1faec73ca..77d2a2053 100644 --- a/receiver/telemetrygeneratorreceiver/config_test.go +++ b/receiver/telemetrygeneratorreceiver/config_test.go @@ -395,12 +395,12 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - valid config", + desc: "metrics - valid config", payloads: 1, errExpected: false, generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -429,13 +429,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid attributes", + desc: "metrics - invalid attributes", payloads: 1, errExpected: true, errText: "error in attributes config: ", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", Attributes: map[string]any{ "attr_key1": struct{}{}, }, @@ -443,13 +443,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid resource attributes", + desc: "metrics - invalid resource attributes", payloads: 1, errExpected: true, errText: "error in resource_attributes config: ", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", ResourceAttributes: map[string]any{ "attr_key1": struct{}{}, }, @@ -457,24 +457,24 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - no metrics", + desc: "metrics - no metrics", payloads: 1, errExpected: true, errText: "metrics must be set", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", }, }, }, { - desc: "host metrics - metrics not array", + desc: "metrics - metrics not array", payloads: 1, errExpected: true, errText: "metrics must be an array of maps", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": map[string]any{ "name": "system.memory.usage", @@ -491,13 +491,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - metric not map", + desc: "metrics - metric not map", payloads: 1, errExpected: true, errText: "each metric must be a map", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ 1, @@ -507,13 +507,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - missing name", + desc: "metrics - missing name", payloads: 1, errExpected: true, errText: "each metric must have a name", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -531,13 +531,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - missing type", + desc: "metrics - missing type", payloads: 1, errExpected: true, errText: "metric system.memory.usage missing type", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -555,13 +555,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid type", + desc: "metrics - invalid type", payloads: 1, errExpected: true, errText: "metric system.memory.usage has invalid metric type: 1", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -580,13 +580,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - unknown type", + desc: "metrics - unknown type", payloads: 1, errExpected: true, errText: "metric system.memory.usage has invalid metric type: Foo", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -605,13 +605,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - missing value_min", + desc: "metrics - missing value_min", payloads: 1, errExpected: true, errText: "metric system.memory.usage missing value_min", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -629,13 +629,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid value_min", + desc: "metrics - invalid value_min", payloads: 1, errExpected: true, errText: "metric system.memory.usage has invalid value_min: foo", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -654,13 +654,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - missing value_max", + desc: "metrics - missing value_max", payloads: 1, errExpected: true, errText: "metric system.memory.usage missing value_max", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -678,13 +678,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid value_max", + desc: "metrics - invalid value_max", payloads: 1, errExpected: true, errText: "metric system.memory.usage has invalid value_max: foo", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -703,13 +703,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - missing unit", + desc: "metrics - missing unit", payloads: 1, errExpected: true, errText: "metric system.memory.usage missing unit", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -727,13 +727,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid unit", + desc: "metrics - invalid unit", payloads: 1, errExpected: true, errText: "metric system.memory.usage has invalid unit: 1", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -752,13 +752,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - unknown unit", + desc: "metrics - unknown unit", payloads: 1, errExpected: true, errText: "metric system.memory.usage has invalid unit: Foo", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -777,13 +777,13 @@ func TestValidate(t *testing.T) { }, }, { - desc: "host metrics - invalid attributes", + desc: "metrics - invalid attributes", payloads: 1, errExpected: true, errText: "error in attributes config for metric system.memory.usage: ", generators: []GeneratorConfig{ { - Type: "host_metrics", + Type: "metrics", AdditionalConfig: map[string]any{ "metrics": []any{ map[string]any{ @@ -801,6 +801,34 @@ func TestValidate(t *testing.T) { }, }, }, + { + desc: "host metrics - invalid attributes", + payloads: 1, + errExpected: true, + errText: "error in resource_attributes config: ", + generators: []GeneratorConfig{ + { + Type: "host_metrics", + ResourceAttributes: map[string]any{ + "state": struct{}{}, + }, + }, + }, + }, + { + desc: "host metrics - valid config", + payloads: 1, + errExpected: false, + generators: []GeneratorConfig{ + { + Type: "host_metrics", + ResourceAttributes: map[string]any{ + "host.name": "2ed77de7e4c1", + "os.type": "linux", + }, + }, + }, + }, } for _, tc := range testCases { diff --git a/receiver/telemetrygeneratorreceiver/generators.go b/receiver/telemetrygeneratorreceiver/generators.go index 0bf520e0a..d875c4ea1 100644 --- a/receiver/telemetrygeneratorreceiver/generators.go +++ b/receiver/telemetrygeneratorreceiver/generators.go @@ -28,6 +28,9 @@ const ( // generatorTypeLogs is the generator type for logs generatorTypeLogs generatorType = "logs" + // generatorTypeMetrics is the generator type for generic metrics + generatorTypeMetrics generatorType = "metrics" + // generatorTypeHostMetrics is the generator type for host metrics generatorTypeHostMetrics generatorType = "host_metrics" @@ -75,6 +78,8 @@ func newMetricsGenerators(cfg *Config, logger *zap.Logger) []metricGenerator { var generators []metricGenerator for _, gen := range cfg.Generators { switch gen.Type { + case generatorTypeMetrics: + generators = append(generators, newMetricsGenerator(gen, logger)) case generatorTypeHostMetrics: generators = append(generators, newHostMetricsGenerator(gen, logger)) case generatorTypeOTLP: diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go index 31b3ee25c..909bffa51 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go @@ -15,116 +15,56 @@ package telemetrygeneratorreceiver //import "github.com/observiq/bindplane-agent/receiver/telemetrygeneratorreceiver" import ( - "math" - "math/rand" - - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/plog" - "go.opentelemetry.io/collector/pdata/pmetric" - "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" ) // hostMetricsGenerator is a generator for host metrics. It generates a sampling of host metrics // emulating the Host Metrics receiver: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver -type hostMetricsGenerator struct { - otlpGenerator - // dataPointUpdaters is a list of functions that update the data points with new values - // for each batch of metrics that are generated - dataPointUpdaters []func() -} -func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerator { - - g := &hostMetricsGenerator{ - otlpGenerator: otlpGenerator{ - cfg: cfg, - logger: logger, - logs: plog.NewLogs(), - metrics: pmetric.NewMetrics(), - traces: ptrace.NewTraces(), +var defaultConfig = GeneratorConfig{ + Type: "host_metrics", + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_unreclaimable"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "cached"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_reclaimable"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "buffered"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "buffered"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_reclaimable"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_unreclaimable"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "cached"}}, + map[string]any{"name": "system.cpu.load_average.1m", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "{thread}"}, + map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "reserved", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, + map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, + map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, + map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, + map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.conntrack.max", "value_min": 65536, "value_max": 65536, "type": "Sum", "unit": "{entries}"}, + map[string]any{"name": "system.network.conntrack.count", "value_min": 8, "value_max": 64, "type": "Sum", "unit": "{entries}"}, + map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "ESTABLISHED"}}, + map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "LISTEN"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "user", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "system", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "idle", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "interrupt", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "nice", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "softirq", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "steal", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "wait", "cpu": "cpu0"}}, }, - } - // Load up Resources attributes - - newResource := g.metrics.ResourceMetrics().AppendEmpty() - err := newResource.Resource().Attributes().FromRaw(cfg.ResourceAttributes) - if err != nil { - // should be caught in validation - logger.Error("Error setting resource attributes in host_metrics", zap.Error(err)) - } - - metrics := cfg.AdditionalConfig["metrics"].([]any) - newScope := newResource.ScopeMetrics().AppendEmpty() - for _, m := range metrics { - metric := m.(map[string]any) - var attributes map[string]any - // attributes are optional - if attr, ok := metric["attributes"]; ok { - attributes = attr.(map[string]any) - } - - metricType := metric["type"].(string) - - name := metric["name"].(string) - valueMin := metric["value_min"].(int) - valueMax := metric["value_max"].(int) - unit := metric["unit"].(string) - - newMetric := newScope.Metrics().AppendEmpty() - newMetric.SetUnit(unit) - newMetric.SetName(name) - - // all the host metrics are either sums or gauges - switch metricType { - case "Gauge": - newMetric.SetEmptyGauge() - dp := newMetric.Gauge().DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - - err = dp.Attributes().FromRaw(attributes) - if err != nil { - // should be caught in validation - logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) - } - - // All the host metric Gauges are float64 - g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(getRandomFloat64(valueMin, valueMax)) }) - case "Sum": - newMetric.SetEmptySum() - dp := newMetric.Sum().DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - err = dp.Attributes().FromRaw(attributes) - if err != nil { - // should be caught in validation - logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) - } - switch unit { - case "s": - g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(float64(math.Trunc(getRandomFloat64(valueMin, valueMax)*100)) / 100) }) - default: - g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetIntValue(int64(getRandomFloat64(valueMin, valueMax))) }) - } - } - } - - g.adjustMetricTimes() - - return g + }, } -// this is a variable so that it can be overridden in tests -var getRandomFloat64 = func(value_min, value_max int) float64 { - // #nosec G404 - we don't need a cryptographically strong random number generator here - return rand.Float64()*(float64(value_max)-float64(value_min)) + float64(value_min) -} - -func (g *hostMetricsGenerator) generateMetrics() pmetric.Metrics { - // Update the data points with new values - for _, updater := range g.dataPointUpdaters { - updater() - } - return g.otlpGenerator.generateMetrics() +func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerator { + // ignore the config, which only contains the type and resource attributes + defaultConfig.ResourceAttributes = cfg.ResourceAttributes + g := newMetricsGenerator(defaultConfig, logger) + return g } diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go index da3bc5060..6fba32509 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go @@ -17,88 +17,27 @@ package telemetrygeneratorreceiver import ( "path/filepath" "testing" - "time" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) var expectedHostMetricsDir = filepath.Join("testdata", "expected_metrics") -func TestHostMetricsGenerator(t *testing.T) { +func TestHostMetricsDefaultConfig(t *testing.T) { - test := []struct { - name string - cfg GeneratorConfig - expectedFile string - }{ - { - name: "default", - cfg: GeneratorConfig{ + // validate the default config + cfg := GeneratorConfig{ - Type: "host_metrics", - ResourceAttributes: map[string]any{ - "host.name": "2ed77de7e4c1", - "os.type": "linux", - }, - AdditionalConfig: map[string]any{ - "metrics": []any{ - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_unreclaimable"}}, - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "cached"}}, - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_reclaimable"}}, - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "buffered"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "buffered"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_reclaimable"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_unreclaimable"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "cached"}}, - map[string]any{"name": "system.cpu.load_average.1m", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "{thread}"}, - map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "reserved", "type": "ext4"}}, - map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, - map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, - map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, - map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, - map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, - map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, - map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, - map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.conntrack.max", "value_min": 65536, "value_max": 65536, "type": "Sum", "unit": "{entries}"}, - map[string]any{"name": "system.network.conntrack.count", "value_min": 8, "value_max": 64, "type": "Sum", "unit": "{entries}"}, - map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "ESTABLISHED"}}, - map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "LISTEN"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "user", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "system", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "idle", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "interrupt", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "nice", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "softirq", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "steal", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "wait", "cpu": "cpu0"}}, - }, - }, - }, - expectedFile: filepath.Join(expectedHostMetricsDir, "host_metrics.yaml"), + // type is intentionally "metrics" because that's what the host_metrics generator is + // using under the hood. This is to ensure that the default configuration is valid, + // since it's not validated at runtime. + Type: "metrics", + ResourceAttributes: map[string]any{ + "host.name": "2ed77de7e4c1", + "os.type": "linux", }, + AdditionalConfig: defaultConfig.AdditionalConfig, } - - for _, tc := range test { - getRandomFloat64 = func(_, _ int) float64 { - return 0 - } - getCurrentTime = func() time.Time { - return time.Unix(0, 0) - } - t.Run(tc.name, func(t *testing.T) { - g := newHostMetricsGenerator(tc.cfg, zap.NewNop()) - metrics := g.generateMetrics() - expectedMetrics, err := golden.ReadMetrics(tc.expectedFile) - require.NoError(t, err) - err = pmetrictest.CompareMetrics(expectedMetrics, metrics) - require.NoError(t, err) - }) - } + err := cfg.Validate() + require.NoError(t, err) } diff --git a/receiver/telemetrygeneratorreceiver/metrics_generator.go b/receiver/telemetrygeneratorreceiver/metrics_generator.go new file mode 100644 index 000000000..0157a18de --- /dev/null +++ b/receiver/telemetrygeneratorreceiver/metrics_generator.go @@ -0,0 +1,130 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetrygeneratorreceiver //import "github.com/observiq/bindplane-agent/receiver/telemetrygeneratorreceiver" + +import ( + "math" + "math/rand" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" +) + +// hostMetricsGenerator is a generator for host metrics. It generates a sampling of host metrics +// emulating the Host Metrics receiver: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver +type metricsGenerator struct { + otlpGenerator + // dataPointUpdaters is a list of functions that update the data points with new values + // for each batch of metrics that are generated + dataPointUpdaters []func() +} + +func newMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerator { + + g := &metricsGenerator{ + otlpGenerator: otlpGenerator{ + cfg: cfg, + logger: logger, + logs: plog.NewLogs(), + metrics: pmetric.NewMetrics(), + traces: ptrace.NewTraces(), + }, + } + // Load up Resources attributes + + newResource := g.metrics.ResourceMetrics().AppendEmpty() + err := newResource.Resource().Attributes().FromRaw(cfg.ResourceAttributes) + if err != nil { + // should be caught in validation + logger.Error("Error setting resource attributes in host_metrics", zap.Error(err)) + } + + metrics := cfg.AdditionalConfig["metrics"].([]any) + newScope := newResource.ScopeMetrics().AppendEmpty() + for _, m := range metrics { + metric := m.(map[string]any) + var attributes map[string]any + // attributes are optional + if attr, ok := metric["attributes"]; ok { + attributes = attr.(map[string]any) + } + + metricType := metric["type"].(string) + + name := metric["name"].(string) + valueMin := metric["value_min"].(int) + valueMax := metric["value_max"].(int) + unit := metric["unit"].(string) + + newMetric := newScope.Metrics().AppendEmpty() + newMetric.SetUnit(unit) + newMetric.SetName(name) + + // all the host metrics are either sums or gauges + switch metricType { + case "Gauge": + newMetric.SetEmptyGauge() + dp := newMetric.Gauge().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + + err = dp.Attributes().FromRaw(attributes) + if err != nil { + // should be caught in validation + logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) + } + + // All the host metric Gauges are float64 + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(getRandomFloat64(valueMin, valueMax)) }) + case "Sum": + newMetric.SetEmptySum() + dp := newMetric.Sum().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) + err = dp.Attributes().FromRaw(attributes) + if err != nil { + // should be caught in validation + logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) + } + switch unit { + case "s": + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(float64(math.Trunc(getRandomFloat64(valueMin, valueMax)*100)) / 100) }) + default: + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetIntValue(int64(getRandomFloat64(valueMin, valueMax))) }) + } + } + } + + g.adjustMetricTimes() + + return g +} + +// this is a variable so that it can be overridden in tests +var getRandomFloat64 = func(value_min, value_max int) float64 { + // #nosec G404 - we don't need a cryptographically strong random number generator here + return rand.Float64()*(float64(value_max)-float64(value_min)) + float64(value_min) +} + +func (g *metricsGenerator) generateMetrics() pmetric.Metrics { + // Update the data points with new values + for _, updater := range g.dataPointUpdaters { + updater() + } + return g.otlpGenerator.generateMetrics() +} diff --git a/receiver/telemetrygeneratorreceiver/metrics_generator_test.go b/receiver/telemetrygeneratorreceiver/metrics_generator_test.go new file mode 100644 index 000000000..662022674 --- /dev/null +++ b/receiver/telemetrygeneratorreceiver/metrics_generator_test.go @@ -0,0 +1,104 @@ +// Copyright observIQ, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package telemetrygeneratorreceiver + +import ( + "path/filepath" + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +var expectedMetricsDir = filepath.Join("testdata", "expected_metrics") + +func TestMetricsGenerator(t *testing.T) { + + test := []struct { + name string + cfg GeneratorConfig + expectedFile string + }{ + { + name: "default", + cfg: GeneratorConfig{ + + Type: "host_metrics", + ResourceAttributes: map[string]any{ + "host.name": "2ed77de7e4c1", + "os.type": "linux", + }, + AdditionalConfig: map[string]any{ + "metrics": []any{ + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_unreclaimable"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "cached"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_reclaimable"}}, + map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "buffered"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "buffered"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_reclaimable"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_unreclaimable"}}, + map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "cached"}}, + map[string]any{"name": "system.cpu.load_average.1m", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "{thread}"}, + map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "reserved", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, + map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, + map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, + map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, + map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, + map[string]any{"name": "system.network.conntrack.max", "value_min": 65536, "value_max": 65536, "type": "Sum", "unit": "{entries}"}, + map[string]any{"name": "system.network.conntrack.count", "value_min": 8, "value_max": 64, "type": "Sum", "unit": "{entries}"}, + map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "ESTABLISHED"}}, + map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "LISTEN"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "user", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "system", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "idle", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "interrupt", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "nice", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "softirq", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "steal", "cpu": "cpu0"}}, + map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "wait", "cpu": "cpu0"}}, + }, + }, + }, + expectedFile: filepath.Join(expectedHostMetricsDir, "metrics.yaml"), + }, + } + + for _, tc := range test { + getRandomFloat64 = func(_, _ int) float64 { + return 0 + } + getCurrentTime = func() time.Time { + return time.Unix(0, 0) + } + t.Run(tc.name, func(t *testing.T) { + g := newMetricsGenerator(tc.cfg, zap.NewNop()) + metrics := g.generateMetrics() + expectedMetrics, err := golden.ReadMetrics(tc.expectedFile) + require.NoError(t, err) + err = pmetrictest.CompareMetrics(expectedMetrics, metrics) + require.NoError(t, err) + }) + } +} diff --git a/receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml b/receiver/telemetrygeneratorreceiver/sample_configs/metrics_config.yaml similarity index 99% rename from receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml rename to receiver/telemetrygeneratorreceiver/sample_configs/metrics_config.yaml index 546b7b627..c942f64c5 100644 --- a/receiver/telemetrygeneratorreceiver/sample_configs/host_metrics_config.yaml +++ b/receiver/telemetrygeneratorreceiver/sample_configs/metrics_config.yaml @@ -1,7 +1,7 @@ telemetrygeneratorreceiver: payloads_per_second: 1 generators: - - type: host_metrics + - type: metrics resource_attributes: host.name: 2ed77de7e4c1 os.type: linux diff --git a/receiver/telemetrygeneratorreceiver/testdata/expected_metrics/host_metrics.yaml b/receiver/telemetrygeneratorreceiver/testdata/expected_metrics/metrics.yaml similarity index 100% rename from receiver/telemetrygeneratorreceiver/testdata/expected_metrics/host_metrics.yaml rename to receiver/telemetrygeneratorreceiver/testdata/expected_metrics/metrics.yaml From 237da23704a8c10b8b3f77c9743b39eb4b834314 Mon Sep 17 00:00:00 2001 From: Sam Hazlehurst Date: Fri, 15 Mar 2024 16:18:27 -0400 Subject: [PATCH 4/4] Use mapstructure to decode config, apply review suggestions --- receiver/telemetrygeneratorreceiver/README.md | 2 +- receiver/telemetrygeneratorreceiver/config.go | 85 +++++--------- .../telemetrygeneratorreceiver/config_test.go | 105 +----------------- receiver/telemetrygeneratorreceiver/go.mod | 1 + receiver/telemetrygeneratorreceiver/go.sum | 2 + .../host_metrics_generator.go | 100 ++++++++++------- .../host_metrics_generator_test.go | 10 +- .../metrics_generator.go | 74 +++++++----- .../metrics_generator_test.go | 6 +- 9 files changed, 146 insertions(+), 239 deletions(-) diff --git a/receiver/telemetrygeneratorreceiver/README.md b/receiver/telemetrygeneratorreceiver/README.md index 16af37704..a4482400d 100644 --- a/receiver/telemetrygeneratorreceiver/README.md +++ b/receiver/telemetrygeneratorreceiver/README.md @@ -3,7 +3,7 @@ This receiver is used to generate synthetic telemetry for testing and configurat ## Minimum Agent Versions - Introduced: [v1.46.0](https://github.com/observIQ/bindplane-agent/releases/tag/v1.46.0) -- Updated to include host_metrics: [v1.47.0](https://github.com/observIQ/bindplane-agent/releases/tag/v1.46.0) +- Updated to include host_metrics: [v1.47.0](https://github.com/observIQ/bindplane-agent/releases/tag/v1.47.0) ## Supported Pipelines - Logs diff --git a/receiver/telemetrygeneratorreceiver/config.go b/receiver/telemetrygeneratorreceiver/config.go index 33231938d..3ef5e4da6 100644 --- a/receiver/telemetrygeneratorreceiver/config.go +++ b/receiver/telemetrygeneratorreceiver/config.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" + "github.com/mitchellh/mapstructure" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -185,12 +186,12 @@ func validateOTLPGenerator(cfg *GeneratorConfig) error { func validateMetricsGeneratorConfig(g *GeneratorConfig) error { err := pcommon.NewMap().FromRaw(g.Attributes) if err != nil { - return fmt.Errorf("error in attributes config: %s", err) + return fmt.Errorf("error in attributes config: %w", err) } err = pcommon.NewMap().FromRaw(g.ResourceAttributes) if err != nil { - return fmt.Errorf("error in resource_attributes config: %s", err) + return fmt.Errorf("error in resource_attributes config: %w", err) } // validate individual metrics @@ -198,6 +199,7 @@ func validateMetricsGeneratorConfig(g *GeneratorConfig) error { if !ok { return errors.New("metrics must be set") } + // check that the metricsArray is a valid array of maps[string]any // Because of the way the config is unmarshaled, we have to use the `[]any` type // and then cast to the correct type @@ -206,81 +208,48 @@ func validateMetricsGeneratorConfig(g *GeneratorConfig) error { return errors.New("metrics must be an array of maps") } for _, m := range metricsArray { - metric, ok := m.(map[string]any) + var metric metric + mapMetric, ok := m.(map[string]any) if !ok { return errors.New("each metric must be a map") } - // check that the metric has a name - name, ok := metric["name"] - if !ok { - return errors.New("each metric must have a name") - } - // check that the metric has a type - metricType, ok := metric["type"] - if !ok { - return fmt.Errorf("metric %s missing type", name) + + err := mapstructure.Decode(mapMetric, &metric) + if err != nil { + return fmt.Errorf("error decoding metric: %w", err) } - // check that the metric type is valid - metricTypeStr, ok := metricType.(string) - if !ok { - return fmt.Errorf("metric %s has invalid metric type: %v", name, metricType) + if metric.Name == "" { + return errors.New("each metric must have a name") } - switch metricTypeStr { + + // validate the metric type + switch metric.Type { case "Gauge", "Sum": + case "": + return fmt.Errorf("metric %s missing type", metric.Name) default: - return fmt.Errorf("metric %s has invalid metric type: %s", name, metricTypeStr) - } - // check that the metric has a value_min - valMin, ok := metric["value_min"] - if !ok { - return fmt.Errorf("metric %s missing value_min", name) + return fmt.Errorf("metric %s has invalid metric type: %s", metric.Name, metric.Type) } - // check that the value_min is a valid int - if _, ok = valMin.(int); !ok { - return fmt.Errorf("metric %s has invalid value_min: %v", name, valMin) - } - // check that the metric has a value_max - valMax, ok := metric["value_max"] - if !ok { - return fmt.Errorf("metric %s missing value_max", name) - } - // check that the value_max is a valid int - if _, ok = valMax.(int); !ok { - return fmt.Errorf("metric %s has invalid value_max: %v", name, valMax) - } - // check that the metric has a unit - unit, ok := metric["unit"] - if !ok { - return fmt.Errorf("metric %s missing unit", name) - } - // check that the unit is a valid string - unitStr, ok := unit.(string) - if !ok { - return fmt.Errorf("metric %s has invalid unit: %v", name, unit) - } - switch unitStr { - case "By", "by", "1", "s", "{thread}", "{errors}", "{packets}", "{entries}", "{connections}", "{faults}", "{operations}", "{processes}": - default: - return fmt.Errorf("metric %s has invalid unit: %s", name, unitStr) + + // validate the unit + if metric.Unit == "" { + return fmt.Errorf("metric %s missing unit", metric.Name) } // attributes are optional - if attr, ok := metric["attributes"]; ok { - attributes := attr.(map[string]any) - err = pcommon.NewMap().FromRaw(attributes) - if err != nil { - return fmt.Errorf("error in attributes config for metric %s: %w", name, err) - } + err = pcommon.NewMap().FromRaw(metric.Attributes) + if err != nil { + return fmt.Errorf("error in attributes config for metric %s: %w", metric.Name, err) } - } + } return nil } func validateHostMetricsGeneratorConfig(g *GeneratorConfig) error { err := pcommon.NewMap().FromRaw(g.ResourceAttributes) if err != nil { - return fmt.Errorf("error in resource_attributes config: %s", err) + return fmt.Errorf("error in resource_attributes config: %w", err) } return nil } diff --git a/receiver/telemetrygeneratorreceiver/config_test.go b/receiver/telemetrygeneratorreceiver/config_test.go index 77d2a2053..420cbac26 100644 --- a/receiver/telemetrygeneratorreceiver/config_test.go +++ b/receiver/telemetrygeneratorreceiver/config_test.go @@ -7,7 +7,6 @@ // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. @@ -558,7 +557,7 @@ func TestValidate(t *testing.T) { desc: "metrics - invalid type", payloads: 1, errExpected: true, - errText: "metric system.memory.usage has invalid metric type: 1", + errText: "error decoding metric: 1 error(s) decoding:\n\n* 'type' expected type 'string', got unconvertible type 'int', value: '1'", generators: []GeneratorConfig{ { Type: "metrics", @@ -604,35 +603,11 @@ func TestValidate(t *testing.T) { }, }, }, - { - desc: "metrics - missing value_min", - payloads: 1, - errExpected: true, - errText: "metric system.memory.usage missing value_min", - generators: []GeneratorConfig{ - { - Type: "metrics", - AdditionalConfig: map[string]any{ - "metrics": []any{ - map[string]any{ - "name": "system.memory.usage", - "value_max": 1000000000, - "type": "Sum", - "unit": "By", - "attributes": map[string]any{ - "state": "slab_reclaimed", - }, - }, - }, - }, - }, - }, - }, { desc: "metrics - invalid value_min", payloads: 1, errExpected: true, - errText: "metric system.memory.usage has invalid value_min: foo", + errText: "error decoding metric: 1 error(s) decoding:\n\n* 'value_min' expected type 'int64', got unconvertible type 'string', value: 'foo'", generators: []GeneratorConfig{ { Type: "metrics", @@ -653,35 +628,11 @@ func TestValidate(t *testing.T) { }, }, }, - { - desc: "metrics - missing value_max", - payloads: 1, - errExpected: true, - errText: "metric system.memory.usage missing value_max", - generators: []GeneratorConfig{ - { - Type: "metrics", - AdditionalConfig: map[string]any{ - "metrics": []any{ - map[string]any{ - "name": "system.memory.usage", - "value_min": 100000, - "type": "Sum", - "unit": "By", - "attributes": map[string]any{ - "state": "slab_reclaimed", - }, - }, - }, - }, - }, - }, - }, { desc: "metrics - invalid value_max", payloads: 1, errExpected: true, - errText: "metric system.memory.usage has invalid value_max: foo", + errText: "error decoding metric: 1 error(s) decoding:\n\n* 'value_max' expected type 'int64', got unconvertible type 'string', value: 'foo'", generators: []GeneratorConfig{ { Type: "metrics", @@ -726,56 +677,6 @@ func TestValidate(t *testing.T) { }, }, }, - { - desc: "metrics - invalid unit", - payloads: 1, - errExpected: true, - errText: "metric system.memory.usage has invalid unit: 1", - generators: []GeneratorConfig{ - { - Type: "metrics", - AdditionalConfig: map[string]any{ - "metrics": []any{ - map[string]any{ - "name": "system.memory.usage", - "value_min": 100000, - "value_max": 1000000000, - "type": "Sum", - "unit": 1, - "attributes": map[string]any{ - "state": "slab_reclaimed", - }, - }, - }, - }, - }, - }, - }, - { - desc: "metrics - unknown unit", - payloads: 1, - errExpected: true, - errText: "metric system.memory.usage has invalid unit: Foo", - generators: []GeneratorConfig{ - { - Type: "metrics", - AdditionalConfig: map[string]any{ - "metrics": []any{ - map[string]any{ - "name": "system.memory.usage", - "value_min": 100000, - "value_max": 1000000000, - "type": "Sum", - "unit": "Foo", - "attributes": map[string]any{ - "state": "slab_reclaimed", - }, - }, - }, - }, - }, - }, - }, { desc: "metrics - invalid attributes", payloads: 1, diff --git a/receiver/telemetrygeneratorreceiver/go.mod b/receiver/telemetrygeneratorreceiver/go.mod index 7055e95f0..c7ce881f6 100644 --- a/receiver/telemetrygeneratorreceiver/go.mod +++ b/receiver/telemetrygeneratorreceiver/go.mod @@ -24,6 +24,7 @@ require ( github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.1.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/receiver/telemetrygeneratorreceiver/go.sum b/receiver/telemetrygeneratorreceiver/go.sum index f935232e4..1bb03765e 100644 --- a/receiver/telemetrygeneratorreceiver/go.sum +++ b/receiver/telemetrygeneratorreceiver/go.sum @@ -36,6 +36,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go index 909bffa51..e04df6299 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator.go @@ -15,56 +15,72 @@ package telemetrygeneratorreceiver //import "github.com/observiq/bindplane-agent/receiver/telemetrygeneratorreceiver" import ( + "errors" + + "github.com/mitchellh/mapstructure" "go.uber.org/zap" ) // hostMetricsGenerator is a generator for host metrics. It generates a sampling of host metrics // emulating the Host Metrics receiver: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver -var defaultConfig = GeneratorConfig{ - Type: "host_metrics", - AdditionalConfig: map[string]any{ - "metrics": []any{ - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_unreclaimable"}}, - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "cached"}}, - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "slab_reclaimable"}}, - map[string]any{"name": "system.memory.utilization", "value_min": 0, "value_max": 100, "type": "Gauge", "unit": "1", "attributes": map[string]any{"state": "buffered"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "buffered"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_reclaimable"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "slab_unreclaimable"}}, - map[string]any{"name": "system.memory.usage", "value_min": 100000, "value_max": 1000000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"state": "cached"}}, - map[string]any{"name": "system.cpu.load_average.1m", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "{thread}"}, - map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "reserved", "type": "ext4"}}, - map[string]any{"name": "system.filesystem.usage", "value_min": 0, "value_max": 15616700416, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, - map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, - map[string]any{"name": "system.filesystem.utilization", "value_min": 0, "value_max": 1, "type": "Gauge", "unit": "1", "attributes": map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, - map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.packets", "value_min": 0, "value_max": 1000000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, - map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "send"}}, - map[string]any{"name": "system.network.io", "value_min": 0, "value_max": 100000000, "type": "Sum", "unit": "By", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.errors", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{errors}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, - map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "transmit"}}, - map[string]any{"name": "system.network.dropped", "value_min": 0, "value_max": 1000, "type": "Sum", "unit": "{packets}", "attributes": map[string]any{"device": "eth0", "direction": "receive"}}, - map[string]any{"name": "system.network.conntrack.max", "value_min": 65536, "value_max": 65536, "type": "Sum", "unit": "{entries}"}, - map[string]any{"name": "system.network.conntrack.count", "value_min": 8, "value_max": 64, "type": "Sum", "unit": "{entries}"}, - map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "ESTABLISHED"}}, - map[string]any{"name": "system.network.connections", "value_min": 0, "value_max": 64, "type": "Sum", "unit": "{connections}", "attributes": map[string]any{"protocol": "tcp", "state": "LISTEN"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "user", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "system", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "idle", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "interrupt", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "nice", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "softirq", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "steal", "cpu": "cpu0"}}, - map[string]any{"name": "system.cpu.time", "value_min": 0, "value_max": 10000, "type": "Sum", "unit": "s", "attributes": map[string]any{"state": "wait", "cpu": "cpu0"}}, - }, - }, +var hostMetrics = []metric{ + {Name: "system.memory.utilization", ValueMin: 0, ValueMax: 100, Type: "Gauge", Unit: "1", Attributes: map[string]any{"state": "slab_unreclaimable"}}, + {Name: "system.memory.utilization", ValueMin: 0, ValueMax: 100, Type: "Gauge", Unit: "1", Attributes: map[string]any{"state": "cached"}}, + {Name: "system.memory.utilization", ValueMin: 0, ValueMax: 100, Type: "Gauge", Unit: "1", Attributes: map[string]any{"state": "slab_reclaimable"}}, + {Name: "system.memory.utilization", ValueMin: 0, ValueMax: 100, Type: "Gauge", Unit: "1", Attributes: map[string]any{"state": "buffered"}}, + {Name: "system.memory.usage", ValueMin: 100000, ValueMax: 1000000000, Type: "Sum", Unit: "By", Attributes: map[string]any{"state": "buffered"}}, + {Name: "system.memory.usage", ValueMin: 100000, ValueMax: 1000000000, Type: "Sum", Unit: "By", Attributes: map[string]any{"state": "slab_reclaimable"}}, + {Name: "system.memory.usage", ValueMin: 100000, ValueMax: 1000000000, Type: "Sum", Unit: "By", Attributes: map[string]any{"state": "slab_unreclaimable"}}, + {Name: "system.memory.usage", ValueMin: 100000, ValueMax: 1000000000, Type: "Sum", Unit: "By", Attributes: map[string]any{"state": "cached"}}, + {Name: "system.cpu.load_average.1m", ValueMin: 0, ValueMax: 1, Type: "Gauge", Unit: "{thread}"}, + {Name: "system.filesystem.usage", ValueMin: 0, ValueMax: 15616700416, Type: "Sum", Unit: "By", Attributes: map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "reserved", "type": "ext4"}}, + {Name: "system.filesystem.usage", ValueMin: 0, ValueMax: 15616700416, Type: "Sum", Unit: "By", Attributes: map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + {Name: "system.filesystem.utilization", ValueMin: 0, ValueMax: 1, Type: "Gauge", Unit: "1", Attributes: map[string]any{"device": "/dev/vda1", "mode": "rw", "mountpoint": "/etc/hosts", "state": "free", "type": "ext4"}}, + {Name: "system.network.packets", ValueMin: 0, ValueMax: 1000000, Type: "Sum", Unit: "{packets}", Attributes: map[string]any{"device": "eth0", "direction": "receive"}}, + {Name: "system.network.packets", ValueMin: 0, ValueMax: 1000000, Type: "Sum", Unit: "{packets}", Attributes: map[string]any{"device": "eth0", "direction": "send"}}, + {Name: "system.network.io", ValueMin: 0, ValueMax: 100000000, Type: "Sum", Unit: "By", Attributes: map[string]any{"device": "eth0", "direction": "send"}}, + {Name: "system.network.io", ValueMin: 0, ValueMax: 100000000, Type: "Sum", Unit: "By", Attributes: map[string]any{"device": "eth0", "direction": "receive"}}, + {Name: "system.network.errors", ValueMin: 0, ValueMax: 1000, Type: "Sum", Unit: "{errors}", Attributes: map[string]any{"device": "eth0", "direction": "receive"}}, + {Name: "system.network.errors", ValueMin: 0, ValueMax: 1000, Type: "Sum", Unit: "{errors}", Attributes: map[string]any{"device": "eth0", "direction": "transmit"}}, + {Name: "system.network.dropped", ValueMin: 0, ValueMax: 1000, Type: "Sum", Unit: "{packets}", Attributes: map[string]any{"device": "eth0", "direction": "transmit"}}, + {Name: "system.network.dropped", ValueMin: 0, ValueMax: 1000, Type: "Sum", Unit: "{packets}", Attributes: map[string]any{"device": "eth0", "direction": "receive"}}, + {Name: "system.network.conntrack.max", ValueMin: 65536, ValueMax: 65536, Type: "Sum", Unit: "{entries}"}, + {Name: "system.network.conntrack.count", ValueMin: 8, ValueMax: 64, Type: "Sum", Unit: "{entries}"}, + {Name: "system.network.connections", ValueMin: 0, ValueMax: 64, Type: "Sum", Unit: "{connections}", Attributes: map[string]any{"protocol": "tcp", "state": "ESTABLISHED"}}, + {Name: "system.network.connections", ValueMin: 0, ValueMax: 64, Type: "Sum", Unit: "{connections}", Attributes: map[string]any{"protocol": "tcp", "state": "LISTEN"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "user", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "system", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "idle", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "interrupt", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "nice", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "softirq", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "steal", "cpu": "cpu0"}}, + {Name: "system.cpu.time", ValueMin: 0, ValueMax: 10000, Type: "Sum", Unit: "s", Attributes: map[string]any{"state": "wait", "cpu": "cpu0"}}, } func newHostMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerator { - // ignore the config, which only contains the type and resource attributes - defaultConfig.ResourceAttributes = cfg.ResourceAttributes - g := newMetricsGenerator(defaultConfig, logger) + cfg, err := fillHostMetricsConfig(cfg) + if err != nil { + logger.Error("Error filling host metrics config", zap.Error(err)) + } + g := newMetricsGenerator(cfg, logger) return g } + +func fillHostMetricsConfig(cfg GeneratorConfig) (GeneratorConfig, error) { + metrics := make([]any, 0, len(hostMetrics)) + + var errs error + for _, m := range hostMetrics { + var metric map[string]any + err := mapstructure.Decode(m, &metric) + if err != nil { + errs = errors.Join(errs, err) + continue + } + metrics = append(metrics, metric) + } + cfg.AdditionalConfig = map[string]any{"metrics": metrics} + return cfg, errs +} diff --git a/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go b/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go index 6fba32509..4828fe300 100644 --- a/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go +++ b/receiver/telemetrygeneratorreceiver/host_metrics_generator_test.go @@ -15,14 +15,11 @@ package telemetrygeneratorreceiver import ( - "path/filepath" "testing" "github.com/stretchr/testify/require" ) -var expectedHostMetricsDir = filepath.Join("testdata", "expected_metrics") - func TestHostMetricsDefaultConfig(t *testing.T) { // validate the default config @@ -36,8 +33,11 @@ func TestHostMetricsDefaultConfig(t *testing.T) { "host.name": "2ed77de7e4c1", "os.type": "linux", }, - AdditionalConfig: defaultConfig.AdditionalConfig, } - err := cfg.Validate() + + cfg, err := fillHostMetricsConfig(cfg) + require.NoError(t, err) + + err = cfg.Validate() require.NoError(t, err) } diff --git a/receiver/telemetrygeneratorreceiver/metrics_generator.go b/receiver/telemetrygeneratorreceiver/metrics_generator.go index 0157a18de..f72edc5ba 100644 --- a/receiver/telemetrygeneratorreceiver/metrics_generator.go +++ b/receiver/telemetrygeneratorreceiver/metrics_generator.go @@ -18,6 +18,7 @@ import ( "math" "math/rand" + "github.com/mitchellh/mapstructure" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/pmetric" @@ -25,8 +26,9 @@ import ( "go.uber.org/zap" ) -// hostMetricsGenerator is a generator for host metrics. It generates a sampling of host metrics -// emulating the Host Metrics receiver: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver +// metricsGenerator is a generator for metrics. It generates a sampling of metrics +// based on the configuration provided, adjusting the values & times of the metrics +// with each generation. type metricsGenerator struct { otlpGenerator // dataPointUpdaters is a list of functions that update the data points with new values @@ -34,6 +36,16 @@ type metricsGenerator struct { dataPointUpdaters []func() } +// metric is a convenience struct for unmarshalling the metrics config +type metric struct { + Name string `mapstructure:"name"` + Type string `mapstructure:"type"` + ValueMin int64 `mapstructure:"value_min"` + ValueMax int64 `mapstructure:"value_max"` + Unit string `mapstructure:"unit"` + Attributes map[string]any `mapstructure:"attributes"` +} + func newMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerator { g := &metricsGenerator{ @@ -45,67 +57,73 @@ func newMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerato traces: ptrace.NewTraces(), }, } - // Load up Resources attributes newResource := g.metrics.ResourceMetrics().AppendEmpty() - err := newResource.Resource().Attributes().FromRaw(cfg.ResourceAttributes) + + // Load up Resources attributes + resourceMap := pcommon.NewMap() + err := resourceMap.FromRaw(cfg.ResourceAttributes) if err != nil { // should be caught in validation logger.Error("Error setting resource attributes in host_metrics", zap.Error(err)) + } else { + resourceMap.CopyTo(newResource.Resource().Attributes()) } metrics := cfg.AdditionalConfig["metrics"].([]any) newScope := newResource.ScopeMetrics().AppendEmpty() + for _, m := range metrics { - metric := m.(map[string]any) - var attributes map[string]any - // attributes are optional - if attr, ok := metric["attributes"]; ok { - attributes = attr.(map[string]any) + var metric metric + err := mapstructure.Decode(m, &metric) + if err != nil { + // this should be caught in validation + logger.Error("Error decoding metric", zap.Error(err)) + continue } - metricType := metric["type"].(string) - - name := metric["name"].(string) - valueMin := metric["value_min"].(int) - valueMax := metric["value_max"].(int) - unit := metric["unit"].(string) - newMetric := newScope.Metrics().AppendEmpty() - newMetric.SetUnit(unit) - newMetric.SetName(name) + + newMetric.SetUnit(metric.Unit) + newMetric.SetName(metric.Name) // all the host metrics are either sums or gauges - switch metricType { + switch metric.Type { case "Gauge": newMetric.SetEmptyGauge() dp := newMetric.Gauge().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - err = dp.Attributes().FromRaw(attributes) + err = dp.Attributes().FromRaw(metric.Attributes) if err != nil { // should be caught in validation - logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) + logger.Error("Error setting attributes in host_metrics", zap.String("name", metric.Name), zap.Error(err)) } // All the host metric Gauges are float64 - g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(getRandomFloat64(valueMin, valueMax)) }) + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(getRandomFloat64(metric.ValueMin, metric.ValueMax)) }) case "Sum": newMetric.SetEmptySum() dp := newMetric.Sum().DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) dp.SetStartTimestamp(pcommon.NewTimestampFromTime(getCurrentTime())) - err = dp.Attributes().FromRaw(attributes) + + err = dp.Attributes().FromRaw(metric.Attributes) if err != nil { // should be caught in validation - logger.Error("Error setting attributes in host_metrics", zap.String("name", name), zap.Error(err)) + logger.Error("Error setting attributes in host_metrics", zap.String("name", metric.Name), zap.Error(err)) } - switch unit { + + switch metric.Unit { case "s": - g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetDoubleValue(float64(math.Trunc(getRandomFloat64(valueMin, valueMax)*100)) / 100) }) + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { + dp.SetDoubleValue(float64(math.Trunc(getRandomFloat64(metric.ValueMin, metric.ValueMax)*100)) / 100) + }) default: - g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetIntValue(int64(getRandomFloat64(valueMin, valueMax))) }) + g.dataPointUpdaters = append(g.dataPointUpdaters, func() { dp.SetIntValue(int64(getRandomFloat64(metric.ValueMin, metric.ValueMax))) }) } } } @@ -116,7 +134,7 @@ func newMetricsGenerator(cfg GeneratorConfig, logger *zap.Logger) metricGenerato } // this is a variable so that it can be overridden in tests -var getRandomFloat64 = func(value_min, value_max int) float64 { +var getRandomFloat64 = func(value_min, value_max int64) float64 { // #nosec G404 - we don't need a cryptographically strong random number generator here return rand.Float64()*(float64(value_max)-float64(value_min)) + float64(value_min) } diff --git a/receiver/telemetrygeneratorreceiver/metrics_generator_test.go b/receiver/telemetrygeneratorreceiver/metrics_generator_test.go index 662022674..c35822f92 100644 --- a/receiver/telemetrygeneratorreceiver/metrics_generator_test.go +++ b/receiver/telemetrygeneratorreceiver/metrics_generator_test.go @@ -38,7 +38,7 @@ func TestMetricsGenerator(t *testing.T) { name: "default", cfg: GeneratorConfig{ - Type: "host_metrics", + Type: "metrics", ResourceAttributes: map[string]any{ "host.name": "2ed77de7e4c1", "os.type": "linux", @@ -81,12 +81,12 @@ func TestMetricsGenerator(t *testing.T) { }, }, }, - expectedFile: filepath.Join(expectedHostMetricsDir, "metrics.yaml"), + expectedFile: filepath.Join(expectedMetricsDir, "metrics.yaml"), }, } for _, tc := range test { - getRandomFloat64 = func(_, _ int) float64 { + getRandomFloat64 = func(_, _ int64) float64 { return 0 } getCurrentTime = func() time.Time {