Skip to content

Commit

Permalink
translatesfx: add support for multiple SA monitors of the same type (o…
Browse files Browse the repository at this point in the history
…pen-telemetry#1696)

* translatesfx: add support for multiple SA monitors of the same type

* translatesfx: respond to PR feedback
  • Loading branch information
pmcollins authored Jun 21, 2022
1 parent 73f5564 commit 25db70e
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 33 deletions.
65 changes: 65 additions & 0 deletions cmd/translatesfx/translatesfx/component.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package translatesfx

import "fmt"

type component struct {
attrs map[string]interface{}
// baseName has the baseName for what will eventually be the key to this
// component in the config -- e.g. "smartagent/sql" which might end up being
// "smartagent/sql/0"
baseName string
}

type componentCollection []component

// toComponentMap turns a componentCollection into a map such that its keys have a `/<number>`
// suffix for any components with colliding provisional keys
func (cc componentCollection) toComponentMap() map[string]map[string]interface{} {
keyCounts := map[string]int{}
hasMultiKeys := map[string]struct{}{}
for _, c := range cc {
count := keyCounts[c.baseName]
if count > 0 {
hasMultiKeys[c.baseName] = struct{}{}
}
keyCounts[c.baseName] = count + 1
}
keyCounts = map[string]int{}
out := map[string]map[string]interface{}{}
for _, c := range cc {
_, found := hasMultiKeys[c.baseName]
key := c.baseName
if found {
numSeen := keyCounts[c.baseName]
key = fmt.Sprintf("%s/%d", key, numSeen)
keyCounts[c.baseName] = numSeen + 1
}
out[key] = c.attrs
}
return out
}

func saMonitorToStandardReceiver(monitor map[string]interface{}) component {
if excludes, ok := monitor[metricsToExclude]; ok {
delete(monitor, metricsToExclude)
monitor["datapointsToExclude"] = excludes
}
return component{
baseName: "smartagent/" + monitor["type"].(string),
attrs: monitor,
}
}
43 changes: 18 additions & 25 deletions cmd/translatesfx/translatesfx/otel.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,23 +244,23 @@ func translateExporters(sa saCfgInfo, cfg *otelCfg) {
}

func translateMonitors(sa saCfgInfo, cfg *otelCfg) (warnings []error) {
rcReceivers := map[string]map[string]interface{}{}
var standardReceivers, rcReceivers componentCollection
for _, monV := range sa.monitors {
monitor := monV.(map[interface{}]interface{})
receiver, w, isRC := saMonitorToOtelReceiver(monitor, sa.observers)
warnings = append(warnings, w...)
target := cfg.Receivers
if isRC {
target = rcReceivers
}
for k, v := range receiver {
target[k] = v
rcReceivers = append(rcReceivers, receiver)
} else {
standardReceivers = append(standardReceivers, receiver)
}
}

cfg.Receivers = standardReceivers.toComponentMap()
rcReceiverMap := rcReceivers.toComponentMap()
metricsReceivers, tracesReceivers, logsReceivers := receiverLists(cfg.Receivers)

if len(rcReceivers) > 0 {
if len(rcReceiverMap) > 0 {
switch {
case sa.observers == nil:
warnings = append(warnings, errors.New("found Smart Agent discovery rule but no observers"))
Expand All @@ -270,7 +270,7 @@ func translateMonitors(sa saCfgInfo, cfg *otelCfg) (warnings []error) {
obs := saObserverTypeToOtel(sa.observers[0].(map[interface{}]interface{})["type"].(string))
const rc = "receiver_creator"
cfg.Receivers[rc] = map[string]interface{}{
"receivers": rcReceivers,
"receivers": rcReceiverMap,
"watch_observers": []string{obs},
}
metricsReceivers = append(metricsReceivers, rc)
Expand Down Expand Up @@ -486,14 +486,14 @@ func sfxExporter(sa saCfgInfo) map[string]map[string]interface{} {
}

func saMonitorToOtelReceiver(monitor map[interface{}]interface{}, observers []interface{}) (
out map[string]map[string]interface{},
cmp component,
warnings []error,
isReceiverCreator bool,
) {
strm := interfaceMapToStringMap(monitor)
if _, ok := monitor[discoveryRule]; ok {
receiver, w := saMonitorToRCReceiver(strm, observers)
return receiver, w, true
cmp, warnings = saMonitorToRCReceiver(strm, observers)
return cmp, warnings, true
}
return saMonitorToStandardReceiver(strm), nil, false
}
Expand All @@ -514,8 +514,8 @@ func stringMapToInterfaceMap(in map[string]interface{}) map[interface{}]interfac
return out
}

func saMonitorToRCReceiver(monitor map[string]interface{}, observers []interface{}) (out map[string]map[string]interface{}, warnings []error) {
key := "smartagent/" + monitor["type"].(string)
func saMonitorToRCReceiver(monitor map[string]interface{}, observers []interface{}) (cmp component, warnings []error) {
baseName := "smartagent/" + monitor["type"].(string)
dr := monitor[discoveryRule].(string)
rcr, err := discoveryRuleToRCRule(dr, observers)
if err != nil {
Expand All @@ -524,22 +524,15 @@ func saMonitorToRCReceiver(monitor map[string]interface{}, observers []interface
warnings = append(warnings, err)
}
delete(monitor, discoveryRule)
return map[string]map[string]interface{}{
key: {

cmp = component{
baseName: baseName,
attrs: map[string]interface{}{
"rule": rcr,
"config": monitor,
},
}, warnings
}

func saMonitorToStandardReceiver(monitor map[string]interface{}) map[string]map[string]interface{} {
if excludes, ok := monitor[metricsToExclude]; ok {
delete(monitor, metricsToExclude)
monitor["datapointsToExclude"] = excludes
}
return map[string]map[string]interface{}{
"smartagent/" + monitor["type"].(string): monitor,
}
return
}

func saObserversToOtel(observers []interface{}) map[string]interface{} {
Expand Down
62 changes: 54 additions & 8 deletions cmd/translatesfx/translatesfx/otel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package translatesfx

import (
"sort"
"strconv"
"testing"

Expand Down Expand Up @@ -58,12 +59,11 @@ func TestSAToOtelConfig(t *testing.T) {
}

func TestMonitorToReceiver(t *testing.T) {
receiver, w, isRC := saMonitorToOtelReceiver(testvSphereMonitorCfg(), nil)
cmp, w, isRC := saMonitorToOtelReceiver(testvSphereMonitorCfg(), nil)
assert.Nil(t, w)
assert.False(t, isRC)
v, ok := receiver["smartagent/vsphere"]
require.True(t, ok)
assert.Equal(t, "vsphere", v["type"])
assert.Equal(t, "smartagent/vsphere", cmp.baseName)
assert.Equal(t, "vsphere", cmp.attrs["type"])
}

func testvSphereMonitorCfg() map[interface{}]interface{} {
Expand All @@ -76,16 +76,16 @@ func testvSphereMonitorCfg() map[interface{}]interface{} {
}

func TestMonitorToReceiver_Rule(t *testing.T) {
otel, w, isRC := saMonitorToOtelReceiver(map[interface{}]interface{}{
cmp, w, isRC := saMonitorToOtelReceiver(map[interface{}]interface{}{
"type": "redis",
"discoveryRule": `target == "hostport" && container_image =~ "redis" && port == 6379`,
}, nil)
assert.Nil(t, w)
assert.True(t, isRC)
redis := otel["smartagent/redis"]
_, ok := redis["rule"]
assert.Equal(t, "smartagent/redis", cmp.baseName)
_, ok := cmp.attrs["rule"]
require.True(t, ok)
_, ok = redis["config"]
_, ok = cmp.attrs["config"]
require.True(t, ok)
}

Expand Down Expand Up @@ -318,6 +318,52 @@ func TestInfoToOtelConfig_MetricsToExclude_Monitor(t *testing.T) {
}, ex)
}

func TestComponentCollection_Single(t *testing.T) {
cc := componentCollection{{
baseName: "mycomponent",
attrs: map[string]interface{}{"foo": "bar"},
}}
componentMap := cc.toComponentMap()
var keys []string
for k := range componentMap {
keys = append(keys, k)
}
sort.Strings(keys)
assert.Equal(t, []string{"mycomponent"}, keys)
}

func TestComponentCollection_Multiple(t *testing.T) {
cc := componentCollection{{
baseName: "mycomponent",
attrs: map[string]interface{}{"foo": "bar"},
}, {
baseName: "mycomponent",
attrs: map[string]interface{}{"foo": "bar"},
}}
componentMap := cc.toComponentMap()
var keys []string
for k := range componentMap {
keys = append(keys, k)
}
sort.Strings(keys)
assert.Equal(t, []string{"mycomponent/0", "mycomponent/1"}, keys)
}

func TestInfoToOtelConfig_DuplicateMonitors(t *testing.T) {
cfg, _ := yamlToOtelConfig(t, "testdata/sa-duplicate-monitors.yaml")
assert.Equal(t, 2, len(cfg.Receivers))
metrics := cfg.Service.Pipelines["metrics"]
const sa0 = "smartagent/sql/0"
const sa1 = "smartagent/sql/1"
assert.Equal(t, []string{sa0, sa1}, metrics.Receivers)
receiver0 := cfg.Receivers[sa0]
_, found := receiver0["connectionString"]
assert.True(t, found)
assert.Equal(t, 7, len(receiver0))
receiver1 := cfg.Receivers[sa1]
assert.Equal(t, map[any]any{"user": "postgres", "password": "s3cr3t"}, receiver1["params"])
}

func yamlToOtelConfig(t *testing.T, filename string) (out *otelCfg, warnings []error) {
cfg := fromYAML(t, filename)
expanded, vaultPaths, err := expandSA(cfg, "")
Expand Down
33 changes: 33 additions & 0 deletions cmd/translatesfx/translatesfx/testdata/sa-duplicate-monitors.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
signalFxAccessToken: abc123
signalFxRealm: us1

logging:
level: debug

monitors:
- type: sql
host: localhost
port: 5432
dbDriver: postgres
params:
user: postgres
password: s3cr3t
connectionString: 'host={{.host}} port={{.port}} user={{.user}} password={{.password}} sslmode=disable'
queries:
- query: 'SELECT 42 as num'
metrics:
- metricName: "my.num"
valueColumn: "num"
- type: sql
host: localhost
port: 5432
dbDriver: postgres
params:
user: postgres
password: s3cr3t
connectionString: 'host={{.host}} port={{.port}} user={{.user}} password={{.password}} sslmode=disable'
queries:
- query: 'SELECT 110 as eleventy'
metrics:
- metricName: "my.other.num"
valueColumn: "eleventy"

0 comments on commit 25db70e

Please sign in to comment.