Skip to content

Commit

Permalink
Merged in rrk/cass-query-gen-test (pull request timescale#82)
Browse files Browse the repository at this point in the history
Refactor query generation for Cassandra; add tests

Approved-by: Lee Hampton <leejhampton@gmail.com>
  • Loading branch information
RobAtticus committed Aug 3, 2018
2 parents 1c1691a + 485b25d commit f6721e9
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 68 deletions.
6 changes: 3 additions & 3 deletions cmd/tsbs_generate_data/serialize/timescaledb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,17 @@ func TestTimescaleDBSerializerSerialize(t *testing.T) {
{
desc: "a regular Point",
inputPoint: testPointDefault,
output: "tags,host_0,eu-west-1,eu-west-1b\ncpu,1451606400000000000,38.24311829\n",
output: "tags,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b\ncpu,1451606400000000000,38.24311829\n",
},
{
desc: "a regular Point using int as value",
inputPoint: testPointInt,
output: "tags,host_0,eu-west-1,eu-west-1b\ncpu,1451606400000000000,38\n",
output: "tags,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b\ncpu,1451606400000000000,38\n",
},
{
desc: "a regular Point with multiple fields",
inputPoint: testPointMultiField,
output: "tags,host_0,eu-west-1,eu-west-1b\ncpu,1451606400000000000,5000000000,38,38.24311829\n",
output: "tags,hostname=host_0,region=eu-west-1,datacenter=eu-west-1b\ncpu,1451606400000000000,5000000000,38,38.24311829\n",
},
{
desc: "a Point with no tags",
Expand Down
87 changes: 30 additions & 57 deletions cmd/tsbs_generate_queries/databases/cassandra/devops.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"time"

"bitbucket.org/440-labs/tsbs/cmd/tsbs_generate_queries/uses/devops"
"bitbucket.org/440-labs/tsbs/cmd/tsbs_generate_queries/utils"
"bitbucket.org/440-labs/tsbs/query"
)

Expand All @@ -24,18 +25,21 @@ func (d *Devops) GenerateEmptyQuery() query.Query {
return query.NewCassandra()
}

func (d *Devops) getHostWhere(nHosts int) []string {
hostnames := d.GetRandomHosts(nHosts)

func (d *Devops) getHostWhereWithHostnames(hostnames []string) []string {
tagSet := []string{}
for _, hostname := range hostnames {
tag := fmt.Sprintf("hostname=%s", hostname)
tag := "hostname=" + hostname
tagSet = append(tagSet, tag)
}

return tagSet
}

func (d *Devops) getHostWhere(nHosts int) []string {
hostnames := d.GetRandomHosts(nHosts)
return d.getHostWhereWithHostnames(hostnames)
}

// GroupByTime selects the MAX for numMetrics metrics under 'cpu',
// per minute for nhosts hosts,
// e.g. in psuedo-SQL:
Expand All @@ -54,19 +58,10 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t
tagSets = append(tagSets, tagSet)

humanLabel := fmt.Sprintf("Cassandra %d cpu metric(s), random %4d hosts, random %s by 1m", numMetrics, nHosts, timeRange)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "max", metrics, interval, tagSets)
q := qi.(*query.Cassandra)
q.HumanLabel = []byte(humanLabel)
q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, interval.StartString()))

q.AggregationType = []byte("max")
q.MeasurementName = []byte("cpu")
q.FieldName = []byte(strings.Join(metrics, ","))

q.TimeStart = interval.Start
q.TimeEnd = interval.End
q.GroupByDuration = time.Minute

q.TagSets = tagSets
}

// GroupByOrderByLimit populates a query.Query that has a time WHERE clause, that groups by a truncated date, orders by that date, and takes a limit:
Expand All @@ -76,18 +71,12 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t
// LIMIT $LIMIT
func (d *Devops) GroupByOrderByLimit(qi query.Query) {
interval := d.Interval.RandWindow(time.Hour)
interval.Start = d.Interval.Start

humanLabel := "Cassandra max cpu over last 5 min-intervals (random end)"
humanDesc := fmt.Sprintf("%s: %s", humanLabel, d.Interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "max", []string{"usage_user"}, interval, nil)
q := qi.(*query.Cassandra)
q.HumanLabel = []byte(humanLabel)
q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, d.Interval.StartString()))

q.AggregationType = []byte("max")
q.MeasurementName = []byte("cpu")
q.FieldName = []byte("usage_user")

q.TimeStart = d.Interval.Start
q.TimeEnd = interval.End
q.GroupByDuration = time.Minute
q.OrderBy = []byte("timestamp_ns DESC")
q.Limit = 5
Expand All @@ -105,16 +94,9 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
metrics := devops.GetCPUMetricsSlice(numMetrics)

humanLabel := devops.GetDoubleGroupByLabel("Cassandra", numMetrics)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "avg", metrics, interval, nil)
q := qi.(*query.Cassandra)
q.HumanLabel = []byte(humanLabel)
q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, interval.StartString()))

q.AggregationType = []byte("avg")
q.MeasurementName = []byte("cpu")
q.FieldName = []byte(strings.Join(metrics, ","))

q.TimeStart = interval.Start
q.TimeEnd = interval.End
q.GroupByDuration = time.Hour
}

Expand All @@ -133,34 +115,19 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) {
tagSets = append(tagSets, tagSet)

humanLabel := devops.GetMaxAllLabel("Cassandra", nHosts)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "max", devops.GetAllCPUMetrics(), interval, tagSets)
q := qi.(*query.Cassandra)
q.HumanLabel = []byte(humanLabel)
q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, interval.StartString()))

q.AggregationType = []byte("max")
q.MeasurementName = []byte("cpu")
q.FieldName = []byte(strings.Join(devops.GetAllCPUMetrics(), ","))

q.TimeStart = interval.Start
q.TimeEnd = interval.End
q.GroupByDuration = time.Hour

q.TagSets = tagSets
}

// LastPointPerHost finds the last row for every host in the dataset
func (d *Devops) LastPointPerHost(qi query.Query) {
humanLabel := "Cassandra last row per host"
humanDesc := fmt.Sprintf("%s: %s", humanLabel, d.Interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "", devops.GetAllCPUMetrics(), d.Interval, nil)
q := qi.(*query.Cassandra)
q.HumanLabel = []byte(humanLabel)
q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, d.Interval.StartString()))

q.MeasurementName = []byte("cpu")
q.FieldName = []byte(strings.Join(devops.GetAllCPUMetrics(), ","))

q.TimeStart = d.Interval.Start
q.TimeEnd = d.Interval.End

q.ForEveryN = []byte("hostname,1")
}

Expand All @@ -182,18 +149,24 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
}

humanLabel := devops.GetHighCPULabel("Cassandra", nHosts)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "", devops.GetAllCPUMetrics(), interval, tagSets)
q := qi.(*query.Cassandra)
q.GroupByDuration = time.Hour
q.WhereClause = []byte("usage_user,>,90.0")
}

func (d *Devops) fillInQuery(qi query.Query, humanLabel, humanDesc, aggType string, fields []string, interval utils.TimeInterval, tagSets [][]string) {
q := qi.(*query.Cassandra)
q.HumanLabel = []byte(humanLabel)
q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, interval.StartString()))
q.HumanDescription = []byte(humanDesc)

q.AggregationType = []byte("")
q.AggregationType = []byte(aggType)
q.MeasurementName = []byte("cpu")
q.FieldName = []byte(strings.Join(devops.GetAllCPUMetrics(), ","))
q.FieldName = []byte(strings.Join(fields, ","))

q.TimeStart = interval.Start
q.TimeEnd = interval.End
q.GroupByDuration = time.Hour
q.WhereClause = []byte("usage_user,>,90.0")

q.TagSets = tagSets
}
110 changes: 110 additions & 0 deletions cmd/tsbs_generate_queries/databases/cassandra/devops_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package cassandra

import (
"strings"
"testing"
"time"

"bitbucket.org/440-labs/tsbs/query"
)

func TestDevopsGetHostWhereWithHostnames(t *testing.T) {
cases := []struct {
desc string
hostnames []string
want []string
}{
{
desc: "single host",
hostnames: []string{"foo1"},
want: []string{"hostname=foo1"},
},
{
desc: "multi host",
hostnames: []string{"foo1", "foo2"},
want: []string{"hostname=foo1", "hostname=foo2"},
},
}

for _, c := range cases {
d := NewDevops(time.Now(), time.Now(), 10)

got := d.getHostWhereWithHostnames(c.hostnames)
if len(got) != len(c.want) {
t.Errorf("%s: incorrect output len: got %d want %d", c.desc, len(got), len(c.want))
}
for i := range c.want {
if got[i] != c.want[i] {
t.Errorf("%s: incorrect output at %d: got %s want %s", c.desc, i, got[i], c.want[i])
}
}
}
}

func TestDevopsFillInQuery(t *testing.T) {
humanLabel := "this is my label"
humanDesc := "and now my description"
aggType := "sum"
fields := []string{"foo1, foo2"}
tags := [][]string{{"foo=val", "bar=val2"}}
now := time.Now()

d := NewDevops(now, now.Add(time.Nanosecond), 10)
qi := d.GenerateEmptyQuery()
q := qi.(*query.Cassandra)
if len(q.HumanLabel) != 0 {
t.Errorf("empty query has non-zero length human label")
}
if len(q.HumanDescription) != 0 {
t.Errorf("empty query has non-zero length human desc")
}
if len(q.AggregationType) != 0 {
t.Errorf("empty query has non-zero length agg type")
}
if len(q.MeasurementName) != 0 {
t.Errorf("empty query has non-zero length measurement name")
}
if len(q.FieldName) != 0 {
t.Errorf("empty query has non-zero length field name")
}
if len(q.TagSets) != 0 {
t.Errorf("empty query has non-zero length tagset")
}

d.fillInQuery(q, humanLabel, humanDesc, aggType, fields, d.Interval, tags)

if got := string(q.HumanLabel); got != humanLabel {
t.Errorf("filled query mislabeled: got %s want %s", got, humanLabel)
}
if got := string(q.HumanDescription); got != humanDesc {
t.Errorf("filled query mis-described: got %s want %s", got, humanDesc)
}
if got := string(q.AggregationType); got != aggType {
t.Errorf("filled query has wrong agg type: got %s want %s", got, aggType)
}
if got := string(q.MeasurementName); got != "cpu" {
t.Errorf("filled query has wrong measurement name: got %s want %s", got, "cpu")
}
if got := string(q.FieldName); got != strings.Join(fields, ",") {
t.Errorf("filled query has wrong fields: got %s want %s", got, strings.Join(fields, ","))
}
if got := q.TimeStart.UnixNano(); got != now.UnixNano() {
t.Errorf("filled query start time wrong: got %d want %d", got, now.UnixNano())
}
if got := q.TimeEnd.UnixNano(); got != now.UnixNano()+1 {
t.Errorf("filled query end time wrong: got %d want %d", got, now.UnixNano()+1)
}
if got := len(q.TagSets); got != len(tags) {
t.Errorf("filled query has wrong tagset length: got %d want %d", got, len(tags))
}
for i := range tags {
if got := len(q.TagSets[i]); got != len(tags[i]) {
t.Errorf("tag set len %d not equal: got %d want %d", i, got, len(tags[i]))
}
for j := range tags[i] {
if got := q.TagSets[i][j]; got != tags[i][j] {
t.Errorf("tag set at %d,%d incorrect: got %s want %s", i, j, got, tags[i][j])
}
}
}
}
16 changes: 8 additions & 8 deletions cmd/tsbs_generate_queries/databases/influx/devops_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,19 @@ func TestDevopsFillInQuery(t *testing.T) {
}

d.fillInQuery(q, humanLabel, humanDesc, influxql)
if string(q.HumanLabel) != humanLabel {
t.Errorf("filled query mislabeled: got %s want %s", string(q.HumanLabel), humanLabel)
if got := string(q.HumanLabel); got != humanLabel {
t.Errorf("filled query mislabeled: got %s want %s", got, humanLabel)
}
if string(q.HumanDescription) != humanDesc {
t.Errorf("filled query mis-described: got %s want %s", string(q.HumanDescription), humanDesc)
if got := string(q.HumanDescription); got != humanDesc {
t.Errorf("filled query mis-described: got %s want %s", got, humanDesc)
}
if string(q.Method) != "GET" {
t.Errorf("filled query has wrong method: got %s want GET", string(q.Method))
if got := string(q.Method); got != "GET" {
t.Errorf("filled query has wrong method: got %s want GET", got)
}
v := url.Values{}
v.Set("q", influxql)
encoded := v.Encode()
if string(q.Path) != "/query?"+encoded {
t.Errorf("filled query has wrong path: got %s want /query?%s", string(q.Path), encoded)
if got := string(q.Path); got != "/query?"+encoded {
t.Errorf("filled query has wrong path: got %s want /query?%s", got, encoded)
}
}

0 comments on commit f6721e9

Please sign in to comment.