Merged in lee/ints-for-cpu-only (pull request timescale#70)

Change devops CPU-only use case to use ints instead of high-precision floats

Approved-by: RobAtticus NA <rob.kiefer@gmail.com>

cmd/tsbs_generate_data/devops/cpu.go

@@ -1,6 +1,7 @@
 package devops
 
 import (
+	"math"
 	"math/rand"
 	"time"
 
@@ -67,6 +68,9 @@ func (m *CPUMeasurement) ToPoint(p *serialize.Point) {
 	p.SetTimestamp(&m.timestamp)
 
 	for i := range m.distributions {
-		p.AppendField(CPUFieldKeys[i], m.distributions[i].Get())
+		// Use ints for CPU metrics.
+		// The full float64 precision in the distributions list is bad for compression on some systems (e.g., ZFS).
+		// Anything above int precision is also not that common or useful for a devops CPU monitoring use case.
+		p.AppendField(CPUFieldKeys[i], math.Round(m.distributions[i].Get()))
 	}
 }

cmd/tsbs_generate_data/serialize/cassandra_test.go

@@ -21,9 +21,9 @@ func TestCassandraSerializerSerialize(t *testing.T) {
 				tagValues:       [][]byte{[]byte("host_0"), []byte("eu-west-1"), []byte("eu-west-1b")},
 				timestamp:       &now,
 				fieldKeys:       [][]byte{[]byte("usage_guest_nice")},
-				fieldValues:     []interface{}{38.2431182911542820},
+				fieldValues:     []interface{}{float64(38.24311829)},
 			},
-			output: "series_double,cpu,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b,usage_guest_nice,2016-01-01,1451606400000000000,38.2431182911542820\n",
+			output: "series_double,cpu,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b,usage_guest_nice,2016-01-01,1451606400000000000,38.24311829\n",
 		},
 	}
 

cmd/tsbs_generate_data/serialize/timescaledb.go

@@ -3,7 +3,6 @@ package serialize
 import (
 	"fmt"
 	"io"
-	"strconv"
 )
 
 // TimescaleDBSerializer writes a Point in a serialized form for TimescaleDB
@@ -45,26 +44,3 @@ func (s *TimescaleDBSerializer) Serialize(p *Point, w io.Writer) error {
 	_, err = w.Write(buf)
 	return err
 }
-
-func fastFormatAppend(v interface{}, buf []byte) []byte {
-	switch v.(type) {
-	case int:
-		return strconv.AppendInt(buf, int64(v.(int)), 10)
-	case int64:
-		return strconv.AppendInt(buf, v.(int64), 10)
-	case float64:
-		return strconv.AppendFloat(buf, v.(float64), 'f', 16, 64)
-	case float32:
-		return strconv.AppendFloat(buf, float64(v.(float32)), 'f', 16, 32)
-	case bool:
-		return strconv.AppendBool(buf, v.(bool))
-	case []byte:
-		buf = append(buf, v.([]byte)...)
-		return buf
-	case string:
-		buf = append(buf, v.(string)...)
-		return buf
-	default:
-		panic(fmt.Sprintf("unknown field type for %#v", v))
-	}
-}

cmd/tsbs_generate_data/serialize/util.go

@@ -0,0 +1,34 @@
+package serialize
+
+import (
+	"fmt"
+	"strconv"
+)
+
+// Utility function for appending various data types to a byte string
+func fastFormatAppend(v interface{}, buf []byte) []byte {
+	switch v.(type) {
+	case int:
+		return strconv.AppendInt(buf, int64(v.(int)), 10)
+	case int64:
+		return strconv.AppendInt(buf, v.(int64), 10)
+	case float64:
+		// Why -1 ?
+		// From Golang source on genericFtoa (called by AppendFloat): 'Negative precision means "only as much as needed to be exact."'
+		// Using this instead of an exact number for precision ensures we preserve the precision passed in to the function, allowing us
+		// to use different precision for different use cases.
+		return strconv.AppendFloat(buf, v.(float64), 'f', -1, 64)
+	case float32:
+		return strconv.AppendFloat(buf, float64(v.(float32)), 'f', -1, 32)
+	case bool:
+		return strconv.AppendBool(buf, v.(bool))
+	case []byte:
+		buf = append(buf, v.([]byte)...)
+		return buf
+	case string:
+		buf = append(buf, v.(string)...)
+		return buf
+	default:
+		panic(fmt.Sprintf("unknown field type for %#v", v))
+	}
+}

cmd/tsbs_generate_data/serialize/util_test.go

@@ -0,0 +1,85 @@
+package serialize
+
+import (
+	"testing"
+)
+
+func TestFastFormatAppend(t *testing.T) {
+	cases := []struct {
+		desc        string
+		inputString []byte
+		input       interface{}
+		output      []byte
+		shouldPanic bool
+	}{
+		{
+			desc:        "fastFormatAppend should properly append a float64 to a given byte string",
+			inputString: []byte("values,"),
+			input:       float64(29.37),
+			output:      []byte("values,29.37"),
+			shouldPanic: false,
+		},
+		{
+			desc:        "fastFormatAppend should properly append a float32 to a given byte string",
+			inputString: []byte("values,"),
+			input:       float32(29.37),
+			output:      []byte("values,29.37"),
+			shouldPanic: false,
+		},
+		{
+			desc:        "fastFormatAppend should properly append an int to a given byte string",
+			inputString: []byte("values,"),
+			input:       int(29),
+			output:      []byte("values,29"),
+			shouldPanic: false,
+		},
+		{
+			desc:        "fastFormatAppend should properly append a byte string to a given byte string",
+			inputString: []byte("values,"),
+			input:       []byte("bytestring"),
+			output:      []byte("values,bytestring"),
+			shouldPanic: false,
+		},
+		{
+			desc:        "fastFormatAppend should properly append a string to a given byte string",
+			inputString: []byte("values,"),
+			input:       "string",
+			output:      []byte("values,string"),
+			shouldPanic: false,
+		},
+		{
+			desc:        "fastFormatAppend should properly append a boolean to a given byte string",
+			inputString: []byte("values,"),
+			input:       true,
+			output:      []byte("values,true"),
+			shouldPanic: false,
+		},
+		{
+			desc:        "fastFormatAppend should panic when given an unsupported type",
+			inputString: []byte("values,"),
+			input:       []int{},
+			output:      []byte("values,true"),
+			shouldPanic: true,
+		},
+	}
+
+	testPanic := func(input interface{}, inputString []byte, desc string) {
+		defer func() {
+			if r := recover(); r == nil {
+				t.Errorf("%s: did not panic when should", desc)
+			}
+		}()
+		fastFormatAppend(input, inputString)
+	}
+
+	for _, c := range cases {
+		if c.shouldPanic == true {
+			testPanic(c.input, c.inputString, c.desc)
+		} else {
+			got := fastFormatAppend(c.input, c.inputString)
+			if string(got) != string(c.output) {
+				t.Errorf("%s \nOutput incorrect: Want: %s Got: %s", c.desc, c.output, got)
+			}
+		}
+	}
+}