Run performance regression checks in TravisCI

.travis.yml

@@ -9,4 +9,6 @@ go:
 # No need for an install step as all dependencies are vendored.
 install: []
 
-script: ./runtests -r small medium large
+script:
+  - ./runtests -r small medium large
+  - ./perfcheck small medium large

Gopkg.toml

@@ -19,3 +19,7 @@
 [[constraint]]
   name = "github.com/stretchr/testify"
   version = "1.1.4"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/perf"

benchcheck/benchcheck.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Jump Trading
+// Copyright 2018 Jump Trading
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,194 +11,97 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-// This file contains source code adapted from software developed for
-// the Go project (see https://github.com/golang/tools/).
 
 package main
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"os"
-	"sort"
 	"strconv"
-	"text/tabwriter"
-
-	"golang.org/x/tools/benchmark/parse"
-)
+	"strings"
 
-var (
-	theshPercent = flag.Int("threshold", 10, "maximum allowed regression (percent)")
+	"golang.org/x/perf/benchstat"
 )
 
-const usageFooter = `
-Each input file should be from:
-	go test -run=NONE -bench=. > [old,new].txt
-
-Benchmark compares old and new for each benchmark and fails if the new
-benchmarks are some percentage theshold slower.
-`
-
-func threshExceeded(d Delta) bool {
-	p := 100*d.Float64() - 100
-	return p > float64(*theshPercent)
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: benchcheck [options] old.txt new.txt\n")
+	fmt.Fprintf(os.Stderr, "options:\n")
+	flag.PrintDefaults()
+	os.Exit(2)
 }
 
-func main() {
-	os.Exit(runMain())
-}
+var flagAlpha = flag.Float64("alpha", 0.05, "consider change significant if p < `α`")
+var flagDelta = flag.Float64("max-delta", 10, "error if delta % larger than this")
+var flagPrint = flag.Bool("print", false, "print output even when no regression found")
 
-func runMain() int {
-	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, "usage: %s old.txt new.txt\n\n", os.Args[0])
-		flag.PrintDefaults()
-		fmt.Fprint(os.Stderr, usageFooter)
-		os.Exit(2)
-	}
+func main() {
+	log.SetPrefix("benchstat: ")
+	log.SetFlags(0)
+	flag.Usage = usage
 	flag.Parse()
 	if flag.NArg() != 2 {
 		flag.Usage()
 	}
 
-	before := parseFile(flag.Arg(0))
-	after := parseFile(flag.Arg(1))
-
-	cmps, warnings := Correlate(before, after)
-	for _, warn := range warnings {
-		fmt.Fprintln(os.Stderr, warn)
-	}
-	if len(cmps) == 0 {
-		fatal("benchcmp: no repeated benchmarks")
+	c := &benchstat.Collection{
+		Alpha:     *flagAlpha,
+		DeltaTest: benchstat.UTest,
 	}
-	sort.Slice(cmps, func(i, j int) bool {
-		return cmps[i].Before.Ord < cmps[j].Before.Ord
-	})
-
-	w := new(tabwriter.Writer)
-	w.Init(os.Stdout, 0, 0, 5, ' ', 0)
-	defer w.Flush()
-
-	var regressions int
-	var header bool // Has the header has been displayed yet for a given block?
-
-	for _, cmp := range cmps {
-		if !cmp.Measured(parse.NsPerOp) {
-			continue
-		}
-		delta := cmp.DeltaNsPerOp()
-		if threshExceeded(delta) {
-			regressions++
-			if !header {
-				fmt.Fprint(w, "benchmark\told ns/op\tnew ns/op\tdelta\n")
-				header = true
-			}
-			fmt.Fprintf(w, "%s\t%s\t%s\t%s\n",
-				cmp.Name(),
-				formatNs(cmp.Before.NsPerOp),
-				formatNs(cmp.After.NsPerOp),
-				delta.Percent(),
-			)
+	for _, file := range flag.Args() {
+		data, err := ioutil.ReadFile(file)
+		if err != nil {
+			log.Fatal(err)
 		}
+		c.AddConfig(file, data)
 	}
 
-	for _, cmp := range cmps {
-		if !cmp.Measured(parse.MBPerS) {
-			continue
-		}
-		delta := cmp.DeltaMBPerS()
-		if threshExceeded(delta) {
-			regressions++
-			if !header {
-				fmt.Fprint(w, "\nbenchmark\told MB/s\tnew MB/s\tspeedup\n")
-				header = true
-			}
-			fmt.Fprintf(w, "%s\t%.2f\t%.2f\t%s\n", cmp.Name(), cmp.Before.MBPerS, cmp.After.MBPerS, delta.Multiple())
-		}
-	}
+	tables := c.Tables()
 
-	header = false
-	for _, cmp := range cmps {
-		if !cmp.Measured(parse.AllocsPerOp) {
-			continue
-		}
-		delta := cmp.DeltaAllocsPerOp()
-		if threshExceeded(delta) {
-			regressions++
-			if !header {
-				fmt.Fprint(w, "\nbenchmark\told allocs\tnew allocs\tdelta\n")
-				header = true
-			}
-			fmt.Fprintf(w, "%s\t%d\t%d\t%s\n",
-				cmp.Name(),
-				cmp.Before.AllocsPerOp,
-				cmp.After.AllocsPerOp,
-				delta.Percent(),
-			)
-		}
-	}
-
-	header = false
-	for _, cmp := range cmps {
-		if !cmp.Measured(parse.AllocedBytesPerOp) {
-			continue
-		}
-		delta := cmp.DeltaAllocedBytesPerOp()
-		if threshExceeded(delta) {
-			regressions++
-			if !header {
-				fmt.Fprint(w, "\nbenchmark\told bytes\tnew bytes\tdelta\n")
-				header = true
-			}
-			fmt.Fprintf(w, "%s\t%d\t%d\t%s\n",
-				cmp.Name(),
-				cmp.Before.AllocedBytesPerOp,
-				cmp.After.AllocedBytesPerOp,
-				delta.Percent(),
-			)
-		}
+	// Look for problematic benchmarks
+	hasProblem, err := findProblems(tables)
+	if err != nil {
+		log.Fatal(err)
 	}
 
-	w.Flush()
-
-	if regressions > 0 {
+	if hasProblem {
+		fmt.Println("Performance regression found!")
 		fmt.Println()
+		printTables(tables)
+	} else if *flagPrint {
+		printTables(tables)
+	} else {
+		fmt.Println("No performance regressions found")
 	}
-	fmt.Printf("%d benchmarks compared.\n", len(cmps))
-	if regressions > 0 {
-		fmt.Fprintf(os.Stderr, "%d performance regressions detected.\n", regressions)
-		return 1
-	}
-	return 0
-}
 
-func fatal(msg interface{}) {
-	fmt.Fprintln(os.Stderr, msg)
-	os.Exit(1)
+	if hasProblem {
+		os.Exit(1)
+	}
 }
 
-func parseFile(path string) parse.Set {
-	f, err := os.Open(path)
-	if err != nil {
-		fatal(err)
-	}
-	defer f.Close()
-	bb, err := parse.ParseSet(f)
-	if err != nil {
-		fatal(err)
+func findProblems(tables []*benchstat.Table) (bool, error) {
+	for _, table := range tables {
+		for _, row := range table.Rows {
+			if row.Delta == "~" {
+				continue
+			}
+			delta, err := strconv.ParseFloat(strings.TrimRight(row.Delta, "%"), 64)
+			if err != nil {
+				return false, err
+			}
+			if delta > *flagDelta {
+				return true, nil
+			}
+		}
 	}
-	return bb
+	return false, nil
 }
 
-// formatNs formats ns measurements to expose a useful amount of
-// precision. It mirrors the ns precision logic of testing.B.
-func formatNs(ns float64) string {
-	prec := 0
-	switch {
-	case ns < 10:
-		prec = 2
-	case ns < 100:
-		prec = 1
-	}
-	return strconv.FormatFloat(ns, 'f', prec, 64)
+func printTables(tables []*benchstat.Table) {
+	var buf bytes.Buffer
+	benchstat.FormatText(&buf, tables)
+	os.Stdout.Write(buf.Bytes())
 }