Skip to content

Commit

Permalink
Refactor generate_queries devops case to use errs
Browse files Browse the repository at this point in the history 
Previously the devops use case generation code used a call to
log.Fatalf when something went wrong. This makes it awkward to test
error conditions when generating queries from other packages, since
we need a way to (a) replace the unexported call to log.Fatalf and
(b) prevent the runtime from actually quitting.

It is better for the library to actually return errors on calls
that can fail, rather than either fataling or panicking. Now other
packages can handle the errors themselves and also test error
conditions in their packages as well.

This refactor was pruned a bit to bubble the 'panic' up one level
for now. When the actual generation code encounters the error
during normal execution, it will panic. But these are easier to
test for and don't require adding hooks to replace the 'fatal'
path in the original package.
  • Loading branch information
@RobAtticus
RobAtticus committed May 22, 2019
1 parent 3cad7a8 commit 3ba75f6
Show file tree
Hide file tree
Showing 9 changed files with 247 additions and 172 deletions.
25 changes: 18 additions & 7 deletions cmd/tsbs_generate_queries/databases/cassandra/devops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,23 @@ import (
"github.com/timescale/tsbs/query"
)

// TODO: Remove the need for this by continuing to bubble up errors
func panicIfErr(err error) {
if err != nil {
panic(err.Error())
}
}

// Devops produces Cassandra-specific queries for all the devops query types.
type Devops struct {
*devops.Core
}

// NewDevops makes an Devops object ready to generate Queries.
func NewDevops(start, end time.Time, scale int) *Devops {
return &Devops{devops.NewCore(start, end, scale)}
core, err := devops.NewCore(start, end, scale)
panicIfErr(err)
return &Devops{core}
}

// GenerateEmptyQuery returns an empty query.Cassandra
Expand All @@ -36,7 +45,8 @@ func (d *Devops) getHostWhereWithHostnames(hostnames []string) []string {
}

func (d *Devops) getHostWhere(nHosts int) []string {
hostnames := d.GetRandomHosts(nHosts)
hostnames, err := d.GetRandomHosts(nHosts)
panicIfErr(err)
return d.getHostWhereWithHostnames(hostnames)
}

Expand All @@ -51,8 +61,8 @@ func (d *Devops) getHostWhere(nHosts int) []string {
// GROUP BY minute ORDER BY minute ASC
func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
interval := d.Interval.MustRandWindow(timeRange)

metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
tagSet := d.getHostWhere(nHosts)

tagSets := [][]string{}
Expand Down Expand Up @@ -96,8 +106,8 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) {
// GROUP BY hour, hostname ORDER BY hour
func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)

metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)

humanLabel := devops.GetDoubleGroupByLabel("Cassandra", numMetrics)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
Expand Down Expand Up @@ -156,7 +166,8 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
tagSets = append(tagSets, tagSet)
}

humanLabel := devops.GetHighCPULabel("Cassandra", nHosts)
humanLabel, err := devops.GetHighCPULabel("Cassandra", nHosts)
panicIfErr(err)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, "", devops.GetAllCPUMetrics(), interval, tagSets)
q := qi.(*query.Cassandra)
Expand Down
27 changes: 18 additions & 9 deletions cmd/tsbs_generate_queries/databases/clickhouse/devops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ import (
"github.com/timescale/tsbs/query"
)

// TODO: Remove the need for this by continuing to bubble up errors
func panicIfErr(err error) {
if err != nil {
panic(err.Error())
}
}

// Devops produces ClickHouse-specific queries for all the devops query types.
type Devops struct {
*devops.Core
Expand All @@ -17,10 +24,9 @@ type Devops struct {

// NewDevops makes an Devops object ready to generate Queries.
func NewDevops(start, end time.Time, scale int) *Devops {
return &Devops{
devops.NewCore(start, end, scale),
false,
}
core, err := devops.NewCore(start, end, scale)
panicIfErr(err)
return &Devops{core, false}
}

// GenerateEmptyQuery returns an empty query.ClickHouse
Expand Down Expand Up @@ -58,7 +64,8 @@ func (d *Devops) getHostWhereWithHostnames(hostnames []string) string {

// getHostWhereString gets multiple random hostnames and create WHERE SQL statement for these hostnames.
func (d *Devops) getHostWhereString(nhosts int) string {
hostnames := d.GetRandomHosts(nhosts)
hostnames, err := d.GetRandomHosts(nhosts)
panicIfErr(err)
return d.getHostWhereWithHostnames(hostnames)
}

Expand Down Expand Up @@ -128,7 +135,8 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) {
// double-groupby-5
// double-groupby-all
func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)

selectClauses := make([]string, numMetrics)
Expand Down Expand Up @@ -239,7 +247,8 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
interval.End().Format(clickhouseTimeStringFormat),
hostWhereClause)

humanLabel := devops.GetHighCPULabel("ClickHouse", nHosts)
humanLabel, err := devops.GetHighCPULabel("ClickHouse", nHosts)
panicIfErr(err)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
d.fillInQuery(qi, humanLabel, humanDesc, sql)
}
Expand Down Expand Up @@ -308,8 +317,8 @@ func (d *Devops) LastPointPerHost(qi query.Query) {
// single-groupby-5-8-1
func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
interval := d.Interval.MustRandWindow(timeRange)

metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
selectClauses := d.getSelectClausesAggMetrics("max", metrics)

sql := fmt.Sprintf(`
Expand Down
23 changes: 18 additions & 5 deletions cmd/tsbs_generate_queries/databases/influx/devops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,23 @@ import (
"github.com/timescale/tsbs/query"
)

// TODO: Remove the need for this by continuing to bubble up errors
func panicIfErr(err error) {
if err != nil {
panic(err.Error())
}
}

// Devops produces Influx-specific queries for all the devops query types.
type Devops struct {
*devops.Core
}

// NewDevops makes an Devops object ready to generate Queries.
func NewDevops(start, end time.Time, scale int) *Devops {
return &Devops{devops.NewCore(start, end, scale)}
core, err := devops.NewCore(start, end, scale)
panicIfErr(err)
return &Devops{core}
}

// GenerateEmptyQuery returns an empty query.HTTP
Expand All @@ -36,7 +45,8 @@ func (d *Devops) getHostWhereWithHostnames(hostnames []string) string {
}

func (d *Devops) getHostWhereString(nHosts int) string {
hostnames := d.GetRandomHosts(nHosts)
hostnames, err := d.GetRandomHosts(nHosts)
panicIfErr(err)
return d.getHostWhereWithHostnames(hostnames)
}

Expand All @@ -60,7 +70,8 @@ func (d *Devops) getSelectClausesAggMetrics(agg string, metrics []string) []stri
// GROUP BY minute ORDER BY minute ASC
func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
interval := d.Interval.MustRandWindow(timeRange)
metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
selectClauses := d.getSelectClausesAggMetrics("max", metrics)
whereHosts := d.getHostWhereString(nHosts)

Expand Down Expand Up @@ -93,7 +104,8 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) {
// WHERE time >= '$HOUR_START' AND time < '$HOUR_END'
// GROUP BY hour, hostname ORDER BY hour, hostname
func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)
selectClauses := d.getSelectClausesAggMetrics("mean", metrics)

Expand Down Expand Up @@ -147,7 +159,8 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
hostWhereClause = fmt.Sprintf("and %s", d.getHostWhereString(nHosts))
}

humanLabel := devops.GetHighCPULabel("Influx", nHosts)
humanLabel, err := devops.GetHighCPULabel("Influx", nHosts)
panicIfErr(err)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
influxql := fmt.Sprintf("SELECT * from cpu where usage_user > 90.0 %s and time >= '%s' and time < '%s'", hostWhereClause, interval.StartString(), interval.EndString())
d.fillInQuery(qi, humanLabel, humanDesc, influxql)
Expand Down
13 changes: 9 additions & 4 deletions cmd/tsbs_generate_queries/databases/mongo/devops-naive.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ type NaiveDevops struct {

// NewNaiveDevops makes an NaiveDevops object ready to generate Queries.
func NewNaiveDevops(start, end time.Time, scale int) *NaiveDevops {
return &NaiveDevops{devops.NewCore(start, end, scale)}
core, err := devops.NewCore(start, end, scale)
panicIfErr(err)
return &NaiveDevops{core}
}

// GenerateEmptyQuery returns an empty query.Mongo
Expand All @@ -45,8 +47,10 @@ func (d *NaiveDevops) GenerateEmptyQuery() query.Query {
// GROUP BY minute ORDER BY minute ASC
func (d *NaiveDevops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
interval := d.Interval.MustRandWindow(timeRange)
hostnames := d.GetRandomHosts(nHosts)
metrics := devops.GetCPUMetricsSlice(numMetrics)
hostnames, err := d.GetRandomHosts(nHosts)
panicIfErr(err)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)

bucketNano := time.Minute.Nanoseconds()
pipelineQuery := []bson.M{
Expand Down Expand Up @@ -106,7 +110,8 @@ func (d *NaiveDevops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRa
// GROUP BY hour, hostname ORDER BY hour, hostname
func (d *NaiveDevops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)
metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
bucketNano := time.Hour.Nanoseconds()

pipelineQuery := []bson.M{
Expand Down
32 changes: 22 additions & 10 deletions cmd/tsbs_generate_queries/databases/mongo/devops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ import (
"github.com/timescale/tsbs/query"
)

// TODO: Remove the need for this by continuing to bubble up errors
func panicIfErr(err error) {
if err != nil {
panic(err.Error())
}
}

func init() {
// needed for serializing the mongo query to gob
gob.Register([]interface{}{})
Expand All @@ -27,7 +34,9 @@ type Devops struct {

// NewDevops makes an Devops object ready to generate Queries.
func NewDevops(start, end time.Time, scale int) *Devops {
return &Devops{devops.NewCore(start, end, scale)}
core, err := devops.NewCore(start, end, scale)
panicIfErr(err)
return &Devops{core}
}

// GenerateEmptyQuery returns an empty query.Mongo
Expand Down Expand Up @@ -102,9 +111,10 @@ func getTimeFilterDocs(interval *utils.TimeInterval) []interface{} {
// GROUP BY minute ORDER BY minute ASC
func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
interval := d.Interval.MustRandWindow(timeRange)

hostnames := d.GetRandomHosts(nHosts)
metrics := devops.GetCPUMetricsSlice(numMetrics)
hostnames, err := d.GetRandomHosts(nHosts)
panicIfErr(err)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
docs := getTimeFilterDocs(interval)
bucketNano := time.Minute.Nanoseconds()

Expand Down Expand Up @@ -171,7 +181,8 @@ func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange t
// GROUP BY hour ORDER BY hour
func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) {
interval := d.Interval.MustRandWindow(devops.MaxAllDuration)
hostnames := d.GetRandomHosts(nHosts)
hostnames, err := d.GetRandomHosts(nHosts)
panicIfErr(err)
docs := getTimeFilterDocs(interval)
bucketNano := time.Hour.Nanoseconds()
metrics := devops.GetAllCPUMetrics()
Expand Down Expand Up @@ -239,8 +250,8 @@ func (d *Devops) MaxAllCPU(qi query.Query, nHosts int) {
// GROUP BY hour, hostname ORDER BY hour, hostname
func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)

metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
docs := getTimeFilterDocs(interval)
bucketNano := time.Hour.Nanoseconds()

Expand Down Expand Up @@ -320,8 +331,8 @@ func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
// AND (hostname = '$HOST' OR hostname = '$HOST2'...)
func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
interval := d.Interval.MustRandWindow(devops.HighCPUDuration)

hostnames := d.GetRandomHosts(nHosts)
hostnames, err := d.GetRandomHosts(nHosts)
panicIfErr(err)
docs := getTimeFilterDocs(interval)

pipelineQuery := []bson.M{}
Expand Down Expand Up @@ -360,7 +371,8 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
},
})

humanLabel := devops.GetHighCPULabel("Mongo", nHosts)
humanLabel, err := devops.GetHighCPULabel("Mongo", nHosts)
panicIfErr(err)
q := qi.(*query.Mongo)
q.HumanLabel = []byte(humanLabel)
q.BsonDoc = pipelineQuery
Expand Down
23 changes: 18 additions & 5 deletions cmd/tsbs_generate_queries/databases/siridb/devops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,23 @@ import (
"github.com/timescale/tsbs/query"
)

// TODO: Remove the need for this by continuing to bubble up errors
func panicIfErr(err error) {
if err != nil {
panic(err.Error())
}
}

// Devops produces SiriDB-specific queries for all the devops query types.
type Devops struct {
*devops.Core
}

// NewDevops makes an Devops object ready to generate Queries.
func NewDevops(start, end time.Time, scale int) *Devops {
return &Devops{devops.NewCore(start, end, scale)}
core, err := devops.NewCore(start, end, scale)
panicIfErr(err)
return &Devops{core}
}

// GenerateEmptyQuery returns an empty query.SiriDB
Expand All @@ -34,7 +43,8 @@ func (d *Devops) getHostWhereWithHostnames(hostnames []string) string {
}

func (d *Devops) getHostWhereString(nhosts int) string {
hostnames := d.GetRandomHosts(nhosts)
hostnames, err := d.GetRandomHosts(nhosts)
panicIfErr(err)
return d.getHostWhereWithHostnames(hostnames)
}

Expand Down Expand Up @@ -64,7 +74,8 @@ const goTimeFmt = "2006-01-02 15:04:05Z"
// select max(1m) from (`groupHost1` | ...) & (`groupMetric1` | ...) between 'time1' and 'time2'
func (d *Devops) GroupByTime(qi query.Query, nHosts, numMetrics int, timeRange time.Duration) {
interval := d.Interval.MustRandWindow(timeRange)
metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
whereMetrics := d.getMetricWhereString(metrics)
whereHosts := d.getHostWhereString(nHosts)

Expand Down Expand Up @@ -102,7 +113,8 @@ func (d *Devops) GroupByOrderByLimit(qi query.Query) {
// select mean(1h) from (`groupMetric1` | ...) between 'time1' and 'time2'
func (d *Devops) GroupByTimeAndPrimaryTag(qi query.Query, numMetrics int) {
interval := d.Interval.MustRandWindow(devops.DoubleGroupByDuration)
metrics := devops.GetCPUMetricsSlice(numMetrics)
metrics, err := devops.GetCPUMetricsSlice(numMetrics)
panicIfErr(err)
whereMetrics := d.getMetricWhereString(metrics)

humanLabel := devops.GetDoubleGroupByLabel("SiriDB", numMetrics)
Expand Down Expand Up @@ -161,7 +173,8 @@ func (d *Devops) HighCPUForHosts(qi query.Query, nHosts int) {
}
interval := d.Interval.MustRandWindow(devops.HighCPUDuration)

humanLabel := devops.GetHighCPULabel("SiriDB", nHosts)
humanLabel, err := devops.GetHighCPULabel("SiriDB", nHosts)
panicIfErr(err)
humanDesc := fmt.Sprintf("%s: %s", humanLabel, interval.StartString())
siriql := fmt.Sprintf("select filter(> 90) from `usage_user` %s between '%s' and '%s'", whereHosts, interval.StartString(), interval.EndString())
d.fillInQuery(qi, humanLabel, humanDesc, siriql)
Expand Down
Loading

0 comments on commit 3ba75f6

Please sign in to comment.