Merge pull request #1 from influxdata/bsmaldon-strings

Use processor conventions from influxdata#4616

plugins/processors/all/all.go

@@ -6,6 +6,6 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/processors/override"
 	_ "github.com/influxdata/telegraf/plugins/processors/printer"
 	_ "github.com/influxdata/telegraf/plugins/processors/regex"
-	_ "github.com/influxdata/telegraf/plugins/processors/topk"
 	_ "github.com/influxdata/telegraf/plugins/processors/strings"
+	_ "github.com/influxdata/telegraf/plugins/processors/topk"
 )

plugins/processors/strings/README.md

@@ -1,48 +1,83 @@
 # Strings Processor Plugin
 
-The `strings` plugin maps certain go string functions onto tag and field values. If `result_key` parameter is present, it can produce new tags and fields from existing ones.
+The `strings` plugin maps certain go string functions onto measurement, tag, and field values.  Values can be modified in place or stored in another key.
 
-Implemented functions are: Lowercase, Uppercase, Trim, TrimPrefix, TrimSuffix, TrimRight, TrimLeft
+Implemented functions are:
+- lowercase
+- uppercase
+- trim
+- trim_left
+- trim_right
+- trim_prefix
+- trim_suffix
 
 Please note that in this implementation these are processed in the order that they appear above.
 
-Specify the `tag` or `field` that you want processed in each section and optionally a `result_key` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor. Certain functions require an `argument` field to specify how they should process their throughput.
-
-Functions that require an `argument` are: Trim, TrimPrefix, TrimSuffix, TrimRight, TrimLeft
+Specify the `measurement`, `tag` or `field` that you want processed in each section and optionally a `dest` if you want the result stored in a new tag or field. You can specify lots of transformations on data with a single strings processor.
 
 ### Configuration:
 
 ```toml
 [[processors.strings]]
-  namepass = ["uri_stem"]
+  # [[processors.strings.uppercase]]
+  #   tag = "method"
 
-  # Tag and field conversions defined in a separate sub-tables
-  [[processors.strings.lowercase]]
-    ## Tag to change
-    tag = "uri_stem"
+  # [[processors.strings.lowercase]]
+  #   field = "uri_stem"
+  #   dest = "uri_stem_normalised"
 
-  [[processors.strings.lowercase]]
-    ## Multiple tags or fields may be defined
-    tag = "method"
+  ## Convert a tag value to lowercase
+  # [[processors.strings.trim]]
+  #   field = "message"
 
-  [[processors.strings.uppercase]]
-    key = "cs-host"
-    result_key = "cs-host_normalised"
+  # [[processors.strings.trim_left]]
+  #   field = "message"
+  #   cutset = "\t"
+
+  # [[processors.strings.trim_right]]
+  #   field = "message"
+  #   cutset = "\r\n"
 
-  [[processors.strings.trimprefix]]
-    tag = "uri_stem"
-    argument = "/api/"
+  # [[processors.strings.trim_prefix]]
+  #   field = "my_value"
+  #   prefix = "my_"
+
+  # [[processors.strings.trim_suffix]]
+  #   field = "read_count"
+  #   suffix = "_count"
 ```
 
-### Tags:
+#### Trim, TrimLeft, TrimRight
+
+The `trim`, `trim_left`, and `trim_right` functions take an optional parameter: `cutset`.  This value is a string containing the characters to remove from the value.
 
-No tags are applied by this processor.
+#### TrimPrefix, TrimSuffix
 
-### Example Input:
+The `trim_prefix` and `trim_suffix` functions remote the given `prefix` or `suffix`
+respectively from the string.
+
+### Example
+**Config**
+```toml
+[[processors.strings]]
+  [[processors.strings.lowercase]]
+    field = "uri-stem"
+
+  [[processors.strings.trim_prefix]]
+    field = "uri_stem"
+    prefix = "/api/"
+
+  [[processors.strings.uppercase]]
+    field = "cs-host"
+    dest = "cs-host_normalised"
+```
+
+**Input**
 ```
 iis_log,method=get,uri_stem=/API/HealthCheck cs-host="MIXEDCASE_host",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000
 ```
-### Example Output:
+
+**Output**
 ```
 iis_log,method=get,uri_stem=healthcheck cs-host="MIXEDCASE_host",cs-host_normalised="MIXEDCASE_HOST",referrer="-",ident="-",http_version=1.1,agent="UserAgent",resp_bytes=270i 1519652321000000000
 ```

plugins/processors/strings/strings.go

@@ -2,109 +2,194 @@ package strings
 
 import (
 	"strings"
+	"unicode"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/processors"
 )
 
 type Strings struct {
-    Lowercase  []converter
-    Uppercase  []converter
-    Trim       []converter
-    TrimLeft   []converter
-    TrimRight  []converter
-    TrimPrefix []converter
-    TrimSuffix []converter
+	Lowercase  []converter `toml:"lowercase"`
+	Uppercase  []converter `toml:"uppercase"`
+	Trim       []converter `toml:"trim"`
+	TrimLeft   []converter `toml:"trim_left"`
+	TrimRight  []converter `toml:"trim_right"`
+	TrimPrefix []converter `toml:"trim_prefix"`
+	TrimSuffix []converter `toml:"trim_suffix"`
+
+	converters []converter
+	init       bool
 }
 
+type ConvertFunc func(s string) string
+
 type converter struct {
+	Field       string
 	Tag         string
-    Field       string
-	ResultKey   string
-    Argument    string
+	Measurement string
+	Dest        string
+	Cutset      string
+	Suffix      string
+	Prefix      string
+
+	fn ConvertFunc
 }
 
 const sampleConfig = `
-  ## Tag and field conversions defined in a separate sub-tables
-
+  ## Convert a tag value to uppercase
   # [[processors.strings.uppercase]]
   #   tag = "method"
 
+  ## Convert a field value to lowercase and store in a new field
   # [[processors.strings.lowercase]]
   #   field = "uri_stem"
-  #   result_key = "uri_stem_normalised"
+  #   dest = "uri_stem_normalised"
+
+  ## Trim leading and trailing whitespace using the default cutset
+  # [[processors.strings.trim]]
+  #   field = "message"
+
+  ## Trim leading characters in cutset
+  # [[processors.strings.trim_left]]
+  #   field = "message"
+  #   cutset = "\t"
+
+  ## Trim trailing characters in cutset
+  # [[processors.strings.trim_right]]
+  #   field = "message"
+  #   cutset = "\r\n"
+
+  ## Trim the given prefix from the field
+  # [[processors.strings.trim_prefix]]
+  #   field = "my_value"
+  #   prefix = "my_"
+
+  ## Trim the given suffix from the field
+  # [[processors.strings.trim_suffix]]
+  #   field = "read_count"
+  #   suffix = "_count"
 `
 
-func (r *Strings) SampleConfig() string {
+func (s *Strings) SampleConfig() string {
 	return sampleConfig
 }
 
-func (r *Strings) Description() string {
-	return "Transforms tag and field values to lower case"
+func (s *Strings) Description() string {
+	return "Perform string processing on tags, fields, and measurements"
+}
+
+func (c *converter) convertTag(metric telegraf.Metric) {
+	tv, ok := metric.GetTag(c.Tag)
+	if !ok {
+		return
+	}
+
+	dest := c.Tag
+	if c.Dest != "" {
+		dest = c.Dest
+	}
+
+	metric.AddTag(dest, c.fn(tv))
 }
 
-func ApplyFunction(
-        metric telegraf.Metric,
-        c converter,
-        fn func(string) string) {
-
-    if value, ok := metric.Tags()[c.Tag]; ok {
-        metric.AddTag(
-            getKey(c),
-            fn(value),
-        )
-    } else if value, ok := metric.Fields()[c.Field]; ok {
-        switch value := value.(type) {
-        case string:
-            metric.AddField(
-                getKey(c),
-                fn(value),
-            )
-        }
-    }
+func (c *converter) convertField(metric telegraf.Metric) {
+	fv, ok := metric.GetField(c.Field)
+	if !ok {
+		return
+	}
+
+	dest := c.Field
+	if c.Dest != "" {
+		dest = c.Dest
+	}
+
+	if fv, ok := fv.(string); ok {
+		metric.AddField(dest, c.fn(fv))
+	}
 }
 
-func (r *Strings) Apply(in ...telegraf.Metric) []telegraf.Metric {
-	for _, metric := range in {
-		for _, converter := range r.Lowercase {
-            ApplyFunction(metric, converter, strings.ToLower)
+func (c *converter) convertMeasurement(metric telegraf.Metric) {
+	if metric.Name() != c.Measurement {
+		return
+	}
+
+	metric.SetName(c.fn(metric.Name()))
+}
+
+func (c *converter) convert(metric telegraf.Metric) {
+	if c.Field != "" {
+		c.convertField(metric)
+	}
+
+	if c.Tag != "" {
+		c.convertTag(metric)
+	}
+
+	if c.Measurement != "" {
+		c.convertMeasurement(metric)
+	}
+}
+
+func (s *Strings) initOnce() {
+	if s.init {
+		return
+	}
+
+	s.converters = make([]converter, 0)
+	for _, c := range s.Lowercase {
+		c.fn = strings.ToLower
+		s.converters = append(s.converters, c)
+	}
+	for _, c := range s.Uppercase {
+		c.fn = strings.ToUpper
+		s.converters = append(s.converters, c)
+	}
+	for _, c := range s.Trim {
+		if c.Cutset != "" {
+			c.fn = func(s string) string { return strings.Trim(s, c.Cutset) }
+		} else {
+			c.fn = func(s string) string { return strings.TrimFunc(s, unicode.IsSpace) }
 		}
-		for _, converter := range r.Uppercase {
-            ApplyFunction(metric, converter, strings.ToUpper)
+		s.converters = append(s.converters, c)
+	}
+	for _, c := range s.TrimLeft {
+		if c.Cutset != "" {
+			c.fn = func(s string) string { return strings.TrimLeft(s, c.Cutset) }
+		} else {
+			c.fn = func(s string) string { return strings.TrimLeftFunc(s, unicode.IsSpace) }
+		}
+		s.converters = append(s.converters, c)
+	}
+	for _, c := range s.TrimRight {
+		if c.Cutset != "" {
+			c.fn = func(s string) string { return strings.TrimRight(s, c.Cutset) }
+		} else {
+			c.fn = func(s string) string { return strings.TrimRightFunc(s, unicode.IsSpace) }
 		}
-        for _, converter := range r.Trim {
-            ApplyFunction(metric, converter,
-                func(s string) string { return strings.Trim(s, converter.Argument) })
-        }
-        for _, converter := range r.TrimPrefix {
-            ApplyFunction(metric, converter,
-                func(s string) string { return strings.TrimPrefix(s, converter.Argument) })
-        }
-        for _, converter := range r.TrimSuffix{
-            ApplyFunction(metric, converter,
-                func(s string) string { return strings.TrimSuffix(s, converter.Argument) })
-        }
-        for _, converter := range r.TrimRight {
-            ApplyFunction(metric, converter,
-                func(s string) string { return strings.TrimRight(s, converter.Argument) })
-        }
-        for _, converter := range r.TrimLeft {
-            ApplyFunction(metric, converter,
-                func(s string) string { return strings.TrimLeft(s, converter.Argument) })
-        }
+		s.converters = append(s.converters, c)
+	}
+	for _, c := range s.TrimPrefix {
+		c.fn = func(s string) string { return strings.TrimPrefix(s, c.Prefix) }
+		s.converters = append(s.converters, c)
+	}
+	for _, c := range s.TrimSuffix {
+		c.fn = func(s string) string { return strings.TrimSuffix(s, c.Suffix) }
+		s.converters = append(s.converters, c)
 	}
 
-	return in
+	s.init = true
 }
 
-func getKey(c converter) string {
-	if c.ResultKey != "" {
-		return c.ResultKey
-	} else if c.Field != "" {
-        return c.Field
-    } else {
-	    return c.Tag
-    }
+func (s *Strings) Apply(in ...telegraf.Metric) []telegraf.Metric {
+	s.initOnce()
+
+	for _, metric := range in {
+		for _, converter := range s.converters {
+			converter.convert(metric)
+		}
+	}
+
+	return in
 }
 
 func init() {