Merged in rrk/query-gen-cleanup (pull request timescale#48)

Refactor tsbs_generate_query to reduce scaleVar passing & remove unused queries

Approved-by: Lee Hampton <leejhampton@gmail.com>

cmd/tsbs_generate_queries/cassandra_devops_1_metric.go

@@ -15,8 +15,8 @@ type CassandraDevopsSingleMetric struct {
 
 // NewCassandraDevopsSingleMetric produces a new function that produces a new CassandraDevopsSingleMetric
 func NewCassandraDevopsSingleMetric(hosts, hours int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newCassandraDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newCassandraDevopsCommon(start, end, scale)
 		return &CassandraDevopsSingleMetric{
 			CassandraDevops: *underlying,
 			hosts:           hosts,
@@ -26,8 +26,8 @@ func NewCassandraDevopsSingleMetric(hosts, hours int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevopsSingleMetric) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevopsSingleMetric) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
-	d.MaxCPUMetricsByMinute(q, scaleVar, d.hosts, 1, time.Duration(int64(d.hours)*int64(time.Hour)))
+	d.MaxCPUMetricsByMinute(q, d.hosts, 1, time.Duration(int64(d.hours)*int64(time.Hour)))
 	return q
 }

cmd/tsbs_generate_queries/cassandra_devops_5_metrics.go

@@ -15,8 +15,8 @@ type CassandraDevops5Metrics struct {
 
 // NewCassandraDevops5Metrics produces a new function that produces a new CassandraDevops5Metrics
 func NewCassandraDevops5Metrics(hosts, hours int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newCassandraDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newCassandraDevopsCommon(start, end, scale)
 		return &CassandraDevops5Metrics{
 			CassandraDevops: *underlying,
 			hosts:           hosts,
@@ -26,8 +26,8 @@ func NewCassandraDevops5Metrics(hosts, hours int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevops5Metrics) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevops5Metrics) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
-	d.MaxCPUMetricsByMinute(q, scaleVar, d.hosts, 5, time.Duration(int64(d.hours)*int64(time.Hour)))
+	d.MaxCPUMetricsByMinute(q, d.hosts, 5, time.Duration(int64(d.hours)*int64(time.Hour)))
 	return q
 }

cmd/tsbs_generate_queries/cassandra_devops_all_max_cpu.go

@@ -14,8 +14,8 @@ type CassandraDevopsAllMaxCPU struct {
 
 // NewCassandraDevopsAllMaxCPU produces a new function that produces a new CassandraDevopsAllMaxCPU
 func NewCassandraDevopsAllMaxCPU(hosts int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newCassandraDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newCassandraDevopsCommon(start, end, scale)
 		return &CassandraDevopsAllMaxCPU{
 			CassandraDevops: *underlying,
 			hosts:           hosts,
@@ -24,8 +24,8 @@ func NewCassandraDevopsAllMaxCPU(hosts int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevopsAllMaxCPU) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevopsAllMaxCPU) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
-	d.MaxAllCPU(q, scaleVar, d.hosts)
+	d.MaxAllCPU(q, d.hosts)
 	return q
 }

cmd/tsbs_generate_queries/cassandra_devops_common.go

@@ -10,22 +10,16 @@ import (
 
 // CassandraDevops produces Cassandra-specific queries for all the devops query types.
 type CassandraDevops struct {
-	AllInterval TimeInterval
+	*devopsCore
 }
 
 // NewCassandraDevops makes an CassandraDevops object ready to generate Queries.
-func newCassandraDevopsCommon(start, end time.Time) *CassandraDevops {
-	if !start.Before(end) {
-		panic("bad time order")
-	}
-
-	return &CassandraDevops{
-		AllInterval: NewTimeInterval(start, end),
-	}
+func newCassandraDevopsCommon(start, end time.Time, scale int) *CassandraDevops {
+	return &CassandraDevops{newDevopsCore(start, end, scale)}
 }
 
-func (d *CassandraDevops) getHostWhere(scaleVar, nhosts int) []string {
-	hostnames := getRandomHosts(scaleVar, nhosts)
+func (d *CassandraDevops) getHostWhere(nhosts int) []string {
+	hostnames := d.getRandomHosts(nhosts)
 
 	tagSet := []string{}
 	for _, hostname := range hostnames {
@@ -45,10 +39,10 @@ func (d *CassandraDevops) getHostWhere(scaleVar, nhosts int) []string {
 // WHERE (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N')
 // AND time >= '$HOUR_START' AND time < '$HOUR_END'
 // GROUP BY minute ORDER BY minute ASC
-func (d *CassandraDevops) MaxCPUMetricsByMinute(qi query.Query, scaleVar, nHosts, numMetrics int, timeRange time.Duration) {
-	interval := d.AllInterval.RandWindow(timeRange)
+func (d *CassandraDevops) MaxCPUMetricsByMinute(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
+	interval := d.interval.RandWindow(timeRange)
 	metrics := getCPUMetricsSlice(numMetrics)
-	tagSet := d.getHostWhere(scaleVar, nHosts)
+	tagSet := d.getHostWhere(nHosts)
 
 	tagSets := [][]string{}
 	tagSets = append(tagSets, tagSet)
@@ -75,18 +69,18 @@ func (d *CassandraDevops) MaxCPUMetricsByMinute(qi query.Query, scaleVar, nHosts
 // GROUP BY t ORDER BY t DESC
 // LIMIT $LIMIT
 func (d *CassandraDevops) GroupByOrderByLimit(qi query.Query) {
-	interval := d.AllInterval.RandWindow(time.Hour)
+	interval := d.interval.RandWindow(time.Hour)
 
 	humanLabel := "Cassandra max cpu over last 5 min-intervals (rand end)"
 	q := qi.(*query.Cassandra)
 	q.HumanLabel = []byte(humanLabel)
-	q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, d.AllInterval.StartString()))
+	q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, d.interval.StartString()))
 
 	q.AggregationType = []byte("max")
 	q.MeasurementName = []byte("cpu")
 	q.FieldName = []byte("usage_user")
 
-	q.TimeStart = d.AllInterval.Start
+	q.TimeStart = d.interval.Start
 	q.TimeEnd = interval.End
 	q.GroupByDuration = time.Minute
 	q.OrderBy = []byte("timestamp_ns DESC")
@@ -101,7 +95,7 @@ func (d *CassandraDevops) GroupByOrderByLimit(qi query.Query) {
 // WHERE time >= '$HOUR_START' AND time < '$HOUR_END'
 // GROUP BY hour, hostname ORDER BY hour
 func (d *CassandraDevops) MeanCPUMetricsDayByHourAllHostsGroupbyHost(qi query.Query, numMetrics int) {
-	interval := d.AllInterval.RandWindow(24 * time.Hour)
+	interval := d.interval.RandWindow(24 * time.Hour)
 	metrics := cpuMetrics[:numMetrics]
 
 	humanLabel := fmt.Sprintf("Cassandra mean of %d metrics, all hosts, rand 1day by 1hr", numMetrics)
@@ -125,9 +119,9 @@ func (d *CassandraDevops) MeanCPUMetricsDayByHourAllHostsGroupbyHost(qi query.Qu
 // FROM cpu WHERE (hostname = '$HOSTNAME_1' OR ... OR hostname = '$HOSTNAME_N')
 // AND time >= '$HOUR_START' AND time < '$HOUR_END'
 // GROUP BY hour ORDER BY hour
-func (d *CassandraDevops) MaxAllCPU(qi query.Query, scaleVar, nhosts int) {
-	interval := d.AllInterval.RandWindow(8 * time.Hour)
-	tagSet := d.getHostWhere(scaleVar, nhosts)
+func (d *CassandraDevops) MaxAllCPU(qi query.Query, nhosts int) {
+	interval := d.interval.RandWindow(8 * time.Hour)
+	tagSet := d.getHostWhere(nhosts)
 
 	tagSets := [][]string{}
 	tagSets = append(tagSets, tagSet)
@@ -153,13 +147,13 @@ func (d *CassandraDevops) LastPointPerHost(qi query.Query) {
 	humanLabel := "Cassandra last row per host"
 	q := qi.(*query.Cassandra)
 	q.HumanLabel = []byte(humanLabel)
-	q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, d.AllInterval.StartString()))
+	q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, d.interval.StartString()))
 
 	q.MeasurementName = []byte("cpu")
 	q.FieldName = []byte("usage_user,usage_system,usage_idle,usage_nice,usage_iowait,usage_irq,usage_softirq,usage_steal,usage_guest,usage_guest_nice")
 
-	q.TimeStart = d.AllInterval.Start
-	q.TimeEnd = d.AllInterval.End
+	q.TimeStart = d.interval.Start
+	q.TimeEnd = d.interval.End
 
 	q.ForEveryN = []byte("hostname,1")
 }
@@ -172,9 +166,9 @@ func (d *CassandraDevops) LastPointPerHost(qi query.Query) {
 // WHERE usage_user > 90.0
 // AND time >= '$TIME_START' AND time < '$TIME_END'
 // AND (hostname = '$HOST' OR hostname = '$HOST2'...)
-func (d *CassandraDevops) HighCPUForHosts(qi query.Query, scaleVar, nhosts int) {
-	interval := d.AllInterval.RandWindow(24 * time.Hour)
-	tagSet := d.getHostWhere(scaleVar, nhosts)
+func (d *CassandraDevops) HighCPUForHosts(qi query.Query, nhosts int) {
+	interval := d.interval.RandWindow(24 * time.Hour)
+	tagSet := d.getHostWhere(nhosts)
 
 	tagSets := [][]string{}
 	if len(tagSet) > 0 {
@@ -203,19 +197,3 @@ func (d *CassandraDevops) HighCPUForHosts(qi query.Query, scaleVar, nhosts int)
 
 	q.TagSets = tagSets
 }
-
-//func (d *CassandraDevops) MeanCPUUsageDayByHourAllHostsGroupbyHost(qi query.Query, _ int) {
-//	interval := d.AllInterval.RandWindow(24*time.Hour)
-//
-//	v := url.Values{}
-//	v.Set("db", d.KeyspaceName)
-//	v.Set("q", fmt.Sprintf("SELECT count(usage_user) from cpu where time >= '%s' and time < '%s' group by time(1h)", interval.StartString(), interval.EndString()))
-//
-//	humanLabel := "Cassandra mean cpu, all hosts, rand 1day by 1hour"
-//	q := qi.(*query.Cassandra)
-//	q.HumanLabel = []byte(humanLabel)
-//	q.HumanDescription = []byte(fmt.Sprintf("%s: %s", humanLabel, interval.StartString()))
-//	q.Method = []byte("GET")
-//	q.Path = []byte(fmt.Sprintf("/query?%s", v.Encode()))
-//	q.Body = nil
-//}

cmd/tsbs_generate_queries/cassandra_devops_groupby.go

@@ -14,8 +14,8 @@ type CassandraDevopsGroupby struct {
 
 // NewCassandraDevopsGroupBy produces a function that produces a new CassandraDevopsGroupby for the given parameters
 func NewCassandraDevopsGroupBy(numMetrics int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newCassandraDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newCassandraDevopsCommon(start, end, scale)
 		return &CassandraDevopsGroupby{
 			CassandraDevops: *underlying,
 			numMetrics:      numMetrics,
@@ -24,7 +24,7 @@ func NewCassandraDevopsGroupBy(numMetrics int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevopsGroupby) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevopsGroupby) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
 	d.MeanCPUMetricsDayByHourAllHostsGroupbyHost(q, d.numMetrics)
 	return q

cmd/tsbs_generate_queries/cassandra_devops_groupby_orderby_limit.go

@@ -12,15 +12,15 @@ type CassandraDevopsGroupByOrderByLimit struct {
 }
 
 // NewCassandraDevopsGroupByOrderByLimit returns a new CassandraDevopsGroupByOrderByLimit for given paremeters
-func NewCassandraDevopsGroupByOrderByLimit(start, end time.Time) QueryGenerator {
-	underlying := newCassandraDevopsCommon(start, end)
+func NewCassandraDevopsGroupByOrderByLimit(start, end time.Time, scale int) QueryGenerator {
+	underlying := newCassandraDevopsCommon(start, end, scale)
 	return &CassandraDevopsGroupByOrderByLimit{
 		CassandraDevops: *underlying,
 	}
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevopsGroupByOrderByLimit) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevopsGroupByOrderByLimit) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
 	d.GroupByOrderByLimit(q)
 	return q

cmd/tsbs_generate_queries/cassandra_devops_high_cpu.go

@@ -14,8 +14,8 @@ type CassandraDevopsHighCPU struct {
 
 // NewCassandraDevopsHighCPU produces a new function that produces a new CassandraDevopsHighCPU
 func NewCassandraDevopsHighCPU(hosts int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newCassandraDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newCassandraDevopsCommon(start, end, scale)
 		return &CassandraDevopsHighCPU{
 			CassandraDevops: *underlying,
 			hosts:           hosts,
@@ -24,8 +24,8 @@ func NewCassandraDevopsHighCPU(hosts int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevopsHighCPU) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevopsHighCPU) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
-	d.HighCPUForHosts(q, scaleVar, d.hosts)
+	d.HighCPUForHosts(q, d.hosts)
 	return q
 }

cmd/tsbs_generate_queries/cassandra_devops_lastpoint.go

@@ -12,16 +12,16 @@ type CassandraDevopsLastPointPerHost struct {
 }
 
 // NewCassandraDevopsLastPointPerHost returns a new CassandraDevopsLastPointPerHost for given paremeters
-func NewCassandraDevopsLastPointPerHost(start, end time.Time) QueryGenerator {
-	underlying := newCassandraDevopsCommon(start, end)
+func NewCassandraDevopsLastPointPerHost(start, end time.Time, scale int) QueryGenerator {
+	underlying := newCassandraDevopsCommon(start, end, scale)
 	return &CassandraDevopsLastPointPerHost{
 		CassandraDevops: *underlying,
 	}
 
 }
 
 // Dispatch fills in the query.Query
-func (d *CassandraDevopsLastPointPerHost) Dispatch(scaleVar int) query.Query {
+func (d *CassandraDevopsLastPointPerHost) Dispatch() query.Query {
 	q := query.NewCassandra() // from pool
 	d.LastPointPerHost(q)
 	return q

cmd/tsbs_generate_queries/devops.go

@@ -9,6 +9,23 @@ import (
 	"bitbucket.org/440-labs/influxdb-comparisons/query"
 )
 
+type devopsCore struct {
+	interval TimeInterval
+	scale    int
+}
+
+func newDevopsCore(start, end time.Time, scale int) *devopsCore {
+	if !start.Before(end) {
+		panic("bad time order")
+	}
+
+	return &devopsCore{NewTimeInterval(start, end), scale}
+}
+
+func (d *devopsCore) getRandomHosts(nHosts int) []string {
+	return getRandomHosts(d.scale, nHosts)
+}
+
 var cpuMetrics = []string{
 	"usage_user",
 	"usage_system",
@@ -43,12 +60,12 @@ type Devops interface {
 	MeanCPUMetricsDayByHourAllHostsGroupbyHost(query.Query, int)
 }
 
-func getRandomHosts(scaleVar, nhosts int) []string {
-	if nhosts > scaleVar {
+func getRandomHosts(scale, nhosts int) []string {
+	if nhosts > scale {
 		log.Fatal("nhosts > scaleVar")
 	}
 
-	nn := rand.Perm(scaleVar)[:nhosts]
+	nn := rand.Perm(scale)[:nhosts]
 
 	hostnames := []string{}
 	for _, n := range nn {

cmd/tsbs_generate_queries/influx_devops_1_metric.go

@@ -15,8 +15,8 @@ type InfluxDevopsSingleMetric struct {
 
 // NewInfluxDevopsSingleMetric produces a new function that produces a new InfluxDevopsSingleMetric
 func NewInfluxDevopsSingleMetric(hosts, hours int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newInfluxDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newInfluxDevopsCommon(start, end, scale)
 		return &InfluxDevopsSingleMetric{
 			InfluxDevops: *underlying,
 			hosts:        hosts,
@@ -26,8 +26,8 @@ func NewInfluxDevopsSingleMetric(hosts, hours int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *InfluxDevopsSingleMetric) Dispatch(scaleVar int) query.Query {
+func (d *InfluxDevopsSingleMetric) Dispatch() query.Query {
 	q := query.NewHTTP() // from pool
-	d.MaxCPUMetricsByMinute(q, scaleVar, d.hosts, 1, time.Duration(int64(d.hours)*int64(time.Hour)))
+	d.MaxCPUMetricsByMinute(q, d.hosts, 1, time.Duration(int64(d.hours)*int64(time.Hour)))
 	return q
 }

cmd/tsbs_generate_queries/influx_devops_5_metrics.go

@@ -15,8 +15,8 @@ type InfluxDevops5Metrics struct {
 
 // NewInfluxDevops5Metrics produces a new function that produces a new InfluxDevops5Metrics
 func NewInfluxDevops5Metrics(hosts, hours int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newInfluxDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newInfluxDevopsCommon(start, end, scale)
 		return &InfluxDevops5Metrics{
 			InfluxDevops: *underlying,
 			hosts:        hosts,
@@ -26,8 +26,8 @@ func NewInfluxDevops5Metrics(hosts, hours int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *InfluxDevops5Metrics) Dispatch(scaleVar int) query.Query {
+func (d *InfluxDevops5Metrics) Dispatch() query.Query {
 	q := query.NewHTTP() // from pool
-	d.MaxCPUMetricsByMinute(q, scaleVar, d.hosts, 5, time.Duration(int64(d.hours)*int64(time.Hour)))
+	d.MaxCPUMetricsByMinute(q, d.hosts, 5, time.Duration(int64(d.hours)*int64(time.Hour)))
 	return q
 }

cmd/tsbs_generate_queries/influx_devops_all_max_cpu.go

@@ -14,8 +14,8 @@ type InfluxDevopsAllMaxCPU struct {
 
 // NewInfluxDevopsAllMaxCPU produces a new function that produces a new InfluxDevopsAllMaxCPU
 func NewInfluxDevopsAllMaxCPU(hosts int) QueryGeneratorMaker {
-	return func(start, end time.Time) QueryGenerator {
-		underlying := newInfluxDevopsCommon(start, end)
+	return func(start, end time.Time, scale int) QueryGenerator {
+		underlying := newInfluxDevopsCommon(start, end, scale)
 		return &InfluxDevopsAllMaxCPU{
 			InfluxDevops: *underlying,
 			hosts:        hosts,
@@ -24,8 +24,8 @@ func NewInfluxDevopsAllMaxCPU(hosts int) QueryGeneratorMaker {
 }
 
 // Dispatch fills in the query.Query
-func (d *InfluxDevopsAllMaxCPU) Dispatch(scaleVar int) query.Query {
+func (d *InfluxDevopsAllMaxCPU) Dispatch() query.Query {
 	q := query.NewHTTP() // from pool
-	d.MaxAllCPU(q, scaleVar, d.hosts)
+	d.MaxAllCPU(q, d.hosts)
 	return q
 }