Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GREEDY field templates for the graphite parser #789

Closed
wants to merge 1 commit into from
Closed

GREEDY field templates for the graphite parser #789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions docs/DATA_FORMATS_INPUT.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,21 @@ cpu.usage.idle.us-west 100
=> cpu_usage,region=us-west idle=100
```

The field key can also be derived from the second "half" of the input metric-name by specifying ```field*```:
```toml
templates = [
"measurement.measurement.region.field*"
]
```

would result in the following Graphite -> Telegraf transformation.

```
cpu.usage.us-west.idle.percentage 100
=> cpu_usage,region=us-west idle_percentage=100
```
(This cannot be used in conjunction with "measurement*"!)

#### Filter Templates:

Users can also filter the template(s) to use based on the name of the bucket,
Expand Down
27 changes: 21 additions & 6 deletions plugins/parsers/graphite/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ func (p *GraphiteParser) ApplyTemplate(line string) (string, map[string]string,
type template struct {
tags []string
defaultTags map[string]string
greedyField bool
greedyMeasurement bool
separator string
}
Expand All @@ -248,6 +249,8 @@ func NewTemplate(pattern string, defaultTags map[string]string, separator string
}
if tag == "measurement*" {
template.greedyMeasurement = true
} else if tag == "field*" {
template.greedyField = true
}
}

Expand All @@ -265,14 +268,26 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
var (
measurement []string
tags = make(map[string]string)
field string
field []string
)

// Set any default tags
for k, v := range t.defaultTags {
tags[k] = v
}

// See if an invalid combination has been specified in the template:
for _, tag := range t.tags {
if tag == "measurement*" {
t.greedyMeasurement = true
} else if tag == "field*" {
t.greedyField = true
}
}
if t.greedyField && t.greedyMeasurement {
return "", nil, "", fmt.Errorf("either 'field*' or 'measurement*' can be used in each template (but not both together): %q", strings.Join(t.tags, t.separator))
}

for i, tag := range t.tags {
if i >= len(fields) {
continue
Expand All @@ -281,10 +296,10 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
if tag == "measurement" {
measurement = append(measurement, fields[i])
} else if tag == "field" {
if len(field) != 0 {
return "", nil, "", fmt.Errorf("'field' can only be used once in each template: %q", line)
}
field = fields[i]
field = append(field, fields[i])
} else if tag == "field*" {
field = append(field, fields[i:]...)
break
} else if tag == "measurement*" {
measurement = append(measurement, fields[i:]...)
break
Expand All @@ -293,7 +308,7 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
}
}

return strings.Join(measurement, t.separator), tags, field, nil
return strings.Join(measurement, t.separator), tags, strings.Join(field, t.separator), nil
}

// matcher determines which template should be applied to a given metric
Expand Down
65 changes: 59 additions & 6 deletions plugins/parsers/graphite/parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,20 @@ func TestTemplateApply(t *testing.T) {
measurement: "cpu.load",
tags: map[string]string{"zone": "us-west"},
},
{
test: "conjoined fields",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here you should test that the correct fields get applied.

ie, this should test that measurement=="cpu.util" and field=="idle.percent" (feel free to make a separate test function for testing field templates)

input: "prod.us-west.server01.cpu.util.idle.percent",
template: "env.zone.host.measurement.measurement.field*",
measurement: "cpu.util",
tags: map[string]string{"env": "prod", "zone": "us-west", "host": "server01"},
},
{
test: "multiple fields",
input: "prod.us-west.server01.cpu.util.idle.percent.free",
template: "env.zone.host.measurement.measurement.field.field.reading",
measurement: "cpu.util",
tags: map[string]string{"env": "prod", "zone": "us-west", "host": "server01", "reading": "free"},
},
}

for _, test := range tests {
Expand Down Expand Up @@ -187,6 +201,12 @@ func TestParse(t *testing.T) {
template: "measurement",
err: `field "cpu" time: strconv.ParseFloat: parsing "14199724z57825": invalid syntax`,
},
{
test: "measurement* and field* (invalid)",
input: `prod.us-west.server01.cpu.util.idle.percent 99.99 1419972457825`,
template: "env.zone.host.measurement*.field*",
err: `either 'field*' or 'measurement*' can be used in each template (but not both together): "env.zone.host.measurement*.field*"`,
},
}

for _, test := range tests {
Expand Down Expand Up @@ -574,15 +594,48 @@ func TestApplyTemplateField(t *testing.T) {
}
}

func TestApplyTemplateFieldError(t *testing.T) {
func TestApplyTemplateMultipleFieldsTogether(t *testing.T) {
p, err := NewGraphiteParser("_",
[]string{"current.* measurement.field.field"}, nil)
[]string{"current.* measurement.measurement.field.field"}, nil)
assert.NoError(t, err)

_, _, _, err = p.ApplyTemplate("current.users.logged_in")
if err == nil {
t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s", err,
"'field' can only be used once in each template: current.users.logged_in")
measurement, _, field, err := p.ApplyTemplate("current.users.logged_in.ssh")

assert.Equal(t, "current_users", measurement)

if field != "logged_in_ssh" {
t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
field, "logged_in_ssh")
}
}

func TestApplyTemplateMultipleFieldsApart(t *testing.T) {
p, err := NewGraphiteParser("_",
[]string{"current.* measurement.measurement.field.method.field"}, nil)
assert.NoError(t, err)

measurement, _, field, err := p.ApplyTemplate("current.users.logged_in.ssh.total")

assert.Equal(t, "current_users", measurement)

if field != "logged_in_total" {
t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
field, "logged_in_total")
}
}

func TestApplyTemplateGreedyField(t *testing.T) {
p, err := NewGraphiteParser("_",
[]string{"current.* measurement.measurement.field*"}, nil)
assert.NoError(t, err)

measurement, _, field, err := p.ApplyTemplate("current.users.logged_in")

assert.Equal(t, "current_users", measurement)

if field != "logged_in" {
t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
field, "logged_in")
}
}

Expand Down